Name: Tan Wen Tao Bryan
Admin No: P2214449
Class: DAAA/FT/2B/01

Image Source: Prashant Banerjee, 2019
# Import libraries
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns
import os
# Extra libraries
from collections import Counter
import string
from wordcloud import WordCloud
import re
# Feature Engineering
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.utils import to_categorical
from sklearn.model_selection import train_test_split
# Keras libraries
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam, RMSprop, SGD
from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import Dense, Embedding, LSTM, Bidirectional, Dropout, GRU, SimpleRNN, Attention, Layer
import keras.backend as K
# Evaluation
from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
from bert_score import score
# Warnings
import warnings
warnings.filterwarnings('ignore')
# Check if GPU is available
tf.config.list_physical_devices('GPU')
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
pd.set_option('max_colwidth', 150)
# Load the dataset
file_path = "./Dataset/partB/train.csv"
data = pd.read_csv(file_path)
display(data.head())
| Quotes | |
|---|---|
| 0 | Embrace the beauty of every sunrise; it's a fresh chance to paint your world with joy. |
| 1 | Embrace challenges; they are the stepping stones to your greatest victories. |
| 2 | Embrace the rhythm of life and let it dance through your soul. |
| 3 | Embrace kindness, for it has the power to change the world one heart at a time. |
| 4 | Embrace the journey, for it leads to the destination of your dreams. |
# Description of the dataset
display(data.describe(include='all'))
print(f'Shape of the Dataset: {data.shape}')
print(f"Number of Missing Values: {data['Quotes'].isnull().sum()}")
unique_quotes = data['Quotes'].unique()
print(f"Number of Unique Quotes: {len(unique_quotes)}")
| Quotes | |
|---|---|
| count | 1000 |
| unique | 890 |
| top | Radiate acceptance, and find peace in embracing what is. |
| freq | 5 |
Shape of the Dataset: (1000, 1) Number of Missing Values: 0 Number of Unique Quotes: 890
Observations
# Remove duplicate quotes
data.drop_duplicates(subset='Quotes', keep='first', inplace=True)
print(f"Shape of dataset after removing duplicates: {data.shape}")
Shape of dataset after removing duplicates: (890, 1)
# Concatenate all the quotes into a single string
all_words = " ".join(data['Quotes'])
# Remove any punctuations
all_words = all_words.translate(str.maketrans("", "", string.punctuation))
# Split the string into words
all_words = all_words.split()
# Count the frequency of each word
word_count = Counter(all_words)
# Get the 20 most common words
most_common_words = word_count.most_common(20)
# Prepare data for plotting
words, counts = zip(*most_common_words)
plt.figure(figsize=(8, 6))
plt.bar(words, counts)
plt.title('Top 20 Most Common Words')
plt.xticks(rotation=45)
plt.xlabel('Words')
plt.ylabel('Frequency')
plt.tight_layout()
plt.show()
Observations:
# Number of unique words
print(f"Number of Unique Words: {len(word_count)}")
# Find total amount of words
total_words = len(all_words)
print(f"Total Number of Words: {total_words}")
# Find the percentage of unique words
unique_words = len(word_count) / total_words * 100
print(f"Percentage of Unique Words: {unique_words:.2f}%")
Number of Unique Words: 1220 Total Number of Words: 10600 Percentage of Unique Words: 11.51%
Observations
# Analyse the distribution of sentence lengths
sentence_lengths = data["Quotes"].str.split().str.len()
# Plot the distribution of sentence lengths
plt.figure(figsize=(8,6))
plt.hist(sentence_lengths, bins=30, edgecolor='black')
plt.title('Distribution of Sentence Lengths')
plt.xlabel('Sentence Length (Number of Words)')
plt.ylabel('Frequency')
plt.tight_layout()
plt.show()
Observations
# Generate a word cloud
wordcloud = WordCloud(
width =800, height=400, background_color='white'
).generate(' '.join(all_words))
plt.figure(figsize=(15, 8))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.show()
Observations
def generate_bigrams(words):
return zip(words, words[1:])
# Generate bigrams from list of words
bigram_list = list(generate_bigrams(all_words))
# Count the frequency of each bigram
bigram_freq = Counter(bigram_list)
# Get the 20 most common bigrams
most_common_bigrams = bigram_freq.most_common(20)
# Prepare data for plotting
bigram_words, bigram_counts = zip(*most_common_bigrams)
bigram_words = [" ".join(bigram) for bigram in bigram_words]
plt.figure(figsize=(10, 8))
plt.bar(bigram_words, bigram_counts)
plt.title('Top 20 Most Common Bigrams')
plt.xticks(rotation=45)
plt.ylabel('Frequency')
plt.tight_layout()
plt.show()
Observations
# Convert the quotes into the list
data_list = list(data["Quotes"].values)
print(data_list)
["Embrace the beauty of every sunrise; it's a fresh chance to paint your world with joy.", 'Embrace challenges; they are the stepping stones to your greatest victories.', 'Embrace the rhythm of life and let it dance through your soul.', 'Embrace kindness, for it has the power to change the world one heart at a time.', 'Embrace the journey, for it leads to the destination of your dreams.', 'Embrace your uniqueness, for it is the fingerprint of your soul on the universe.', 'Embrace the present moment, for it is the only one that truly exists.', 'Embrace your fears, for they hold the key to unlocking your true potential.', 'Embrace gratitude, and watch how it multiplies the blessings in your life.', 'Embrace the rain, for it nourishes the seeds of your future success.', 'Embrace the whispers of your heart; they carry the wisdom of the universe.', 'Embrace laughter, for it is the music of a joyful heart.', 'Embrace the power of forgiveness, for it sets you free from the chains of the past.', 'Embrace the silence; it speaks louder than words ever could.', 'Embrace the small moments, for they often hold the greatest significance.', 'Embrace love, for it is the language of the soul.', "Embrace change, for it is the only constant in life's beautiful chaos.", 'Embrace the unknown, for it holds the promise of new beginnings.', 'Embrace your dreams, for they are the blueprints of your destiny.', 'Embrace the colors of life, even in the midst of the darkest days.', 'Embrace compassion, for it is the cornerstone of true strength.', 'Embrace simplicity, for it is the gateway to a peaceful heart.', 'Embrace the power of a genuine smile; it can light up the world.', 'Embrace the wisdom of elders, for it is the treasure chest of experience.', 'Embrace patience, for it allows the universe to unfold in its own time.', 'Embrace the melody of your soul, and let it sing its own song.', 'Embrace the power of imagination; it can turn dreams into reality.', 'Embrace the whispers of the wind; they carry secrets of far-off lands.', 'Embrace vulnerability, for it is the birthplace of true connection.', 'Embrace the dance of life, and let your heart lead the way.', 'Embrace the power of positive thoughts; they have the ability to shape your destiny.', 'Embrace the embrace itself, for it is the language of love without words.', 'Embrace the light within, and let it guide you through the darkest nights.', 'Embrace the symphony of nature, for it is a lullaby for the soul.', 'Embrace the power of gratitude, for it turns what you have into more than enough.', 'Embrace adventure, for it is the heartbeat of a life well-lived.', 'Embrace the pages of your story, for they hold the chapters of your growth.', 'Embrace the whispers of possibility, for they hold the keys to your dreams.', 'Embrace the power of a warm hug; it can heal wounds and mend hearts.', "Embrace the magic in everyday moments; it's the alchemy of a joyful heart.", 'Embrace the symphony of life, and let your heart be the conductor.', 'Embrace the power of positive change, and watch your world transform.', 'Embrace the energy of the universe, and let it flow through your being.', 'Embrace the symphony of stars; they hold the stories of the cosmos.', 'Embrace the power of a kind word; it can light up even the darkest days.', 'Embrace the whispers of intuition, for they are the compass of your soul.', 'Embrace the dance of shadows, for they remind you of the brilliance of light.', 'Embrace the colors of your emotions, for they paint the canvas of your life.', 'Embrace the power of community, for it is the heartbeat of humanity.', 'Embrace the rhythm of your heart, and let it lead you to your passions.', "Embrace the magic in the ordinary; it's the heartbeat of a grateful heart.", 'Embrace the symphony of laughter, for it is the music of the soul.', 'Embrace the power of learning, for it is the key to an ever-expanding mind.', 'Embrace the whispers of the moon; they hold the secrets of the night.', 'Embrace the dance of seasons, for they mirror the cycles of life.', 'Embrace the colors of diversity, for they weave the tapestry of humanity.', 'Embrace the power of a loving touch; it can heal wounds that words cannot.', 'Embrace the symphony of dreams, for they are the blueprints of your destiny.', 'Embrace the power of positive affirmations; they are the seeds of transformation.', 'Embrace the whispers of hope, for they carry the promise of a brighter tomorrow.', 'Embrace the dance of solitude, for it is where you find the truest reflection of self.', 'Embrace the colors of creativity, for they paint the masterpiece of your life.', 'Embrace the power of presence; it is the greatest gift you can give to another.', 'Embrace the symphony of raindrops; they cleanse the spirit and nurture the soul.', 'Embrace the power of purpose, for it is the driving force behind a fulfilled life.', 'Embrace the whispers of inspiration, for they hold the keys to your potential.', 'Embrace the dance of forgiveness, for it sets your heart free from burdens.', 'Embrace the colors of forgiveness, for they bring healing to wounded souls.', 'Embrace the power of a warm embrace; it can mend hearts and restore faith.', 'Embrace the symphony of gratitude, for it is the music of a joyful heart.', 'Embrace the power of positivity, for it is the key to unlocking a bright future.', 'Embrace the whispers of courage; they hold the strength to overcome any challenge.', 'Embrace the dance of gratitude, for it multiplies the blessings in your life.', 'Embrace the colors of empathy, for they paint the world with compassion.', 'Embrace the power of self-love; it is the foundation of a joyful heart.', 'Embrace the symphony of growth; it is the melody of a life well-lived.', 'Embrace the power of self-discovery; it leads you to the treasure within.', 'Embrace the whispers of faith, for they carry the promise of brighter days.', 'Embrace the dance of self-acceptance, for it is where true beauty resides.', 'Embrace the colors of resilience, for they paint the portrait of your strength.', 'Embrace the power of positive energy; it radiates from your heart to the world.', 'Embrace the symphony of possibilities, for they are the seeds of your dreams.', 'Embrace the power of connection, for it is the heartbeat of love.', 'Embrace the dance of self-care, for it nurtures your body, mind, and soul.', 'Embrace the colors of authenticity, for they paint the canvas of your true self.', 'Embrace the power of a kind deed; it can transform lives and inspire change.', 'Embrace the symphony of joy; it is the music of a heart at peace.', 'Embrace the power of a grateful heart, for it turns ordinary moments into blessings.', 'Embrace the whispers of peace, for they bring solace to a restless soul.', 'Embrace the dance of balance, for it is the key to a harmonious life.', 'Embrace the colors of adventure, for they paint the tapestry of your journey.', 'Embrace the power of self-expression, for it is the voice of your soul.', 'Embrace the symphony of abundance, for it is the melody of a fulfilled life.', 'Embrace the power of a warm smile; it can light up even the gloomiest days.', 'Embrace the whispers of possibility, for they carry the seeds of your potential.', 'Embrace the dance of gratitude, for it is the heartbeat of a joyful heart.', 'Embrace the colors of trust, for they paint the masterpiece of your relationships.', 'Embrace the power of presence, for it is the most precious gift you can give.', 'Embrace the symphony of life, and let your heart be the conductor of your destiny.', 'Radiate positivity, and watch how it illuminates your world.', 'Radiate kindness, for it has the power to transform hearts and minds.', 'Radiate love, for it is the most powerful force in the universe.', "Radiate gratitude, and you'll attract more blessings into your life.", 'Radiate confidence, for it is the key to unlocking your true potential.', 'Radiate joy, and let it light up the lives of those around you.', 'Radiate authenticity, for it is the essence of true beauty.', 'Radiate peace, and create a sanctuary of serenity within you.', 'Radiate resilience, and let it be the armor that shields your heart.', 'Radiate wisdom, for it carries the light of experience.', 'Radiate forgiveness, for it sets your spirit free from bitterness.', 'Radiate compassion, for it is the language of the heart.', 'Radiate strength, for it is the foundation of your inner power.', 'Radiate hope, for it brings light to even the darkest of days.', 'Radiate curiosity, and let it lead you to new adventures.', 'Radiate humility, for it is the mark of a truly great soul.', 'Radiate determination, and let it fuel your journey to success.', 'Radiate acceptance, and find peace in embracing what is.', 'Radiate enthusiasm, and watch how it ignites the world around you.', 'Radiate positivity, for it is contagious and boundless.', 'Radiate grace, for it is the mark of a truly beautiful soul.', 'Radiate gratitude, for it turns what you have into more than enough.', 'Radiate kindness, and become a beacon of light in the world.', "Radiate joy, and let it be the melody of your life's song.", 'Radiate love, for it is the truest expression of your essence.', 'Radiate confidence, and watch how it propels you towards your dreams.', 'Radiate authenticity, for it is the essence of true connection.', 'Radiate peace, and create a haven of tranquility around you.', 'Radiate resilience, and let it be the foundation of your strength.', 'Radiate wisdom, for it carries the brilliance of a well-lived life.', 'Radiate forgiveness, and set your heart free from the weight of grudges.', 'Radiate compassion, for it is the language of the soul.', 'Radiate strength, and let it be the cornerstone of your character.', 'Radiate hope, and let it be the beacon that guides your way.', 'Radiate curiosity, for it is the spark of endless discovery.', 'Radiate humility, and let it be the foundation of your greatness.', 'Radiate enthusiasm, and watch how it spreads like wildfire.', 'Radiate positivity, for it is a gift that keeps on giving.', 'Radiate grace, and let it be the signature of your presence.', 'Radiate gratitude, for it turns even the simplest moments into treasures.', 'Radiate kindness, and become a force of good in the world.', 'Radiate joy, and let it be the guiding star of your life.', 'Radiate love, for it is the truest expression of your being.', 'Radiate confidence, and let it be the wind in your sails.', 'Radiate authenticity, for it is the foundation of genuine connection.', 'Radiate peace, and let it be the sanctuary within your heart.', 'Radiate resilience, and let it be the fortress of your spirit.', 'Radiate wisdom, for it carries the echoes of a life well-lived.', 'Radiate forgiveness, and let it be the balm that heals old wounds.', 'Radiate compassion, for it is the heartbeat of humanity.', 'Radiate strength, and let it be the anchor in the storms of life.', 'Radiate hope, and let it be the sunrise in your darkest nights.', 'Radiate curiosity, for it is the compass of endless exploration.', 'Radiate humility, and let it be the foundation of your legacy.', 'Radiate determination, and let it be the driving force of your journey.', 'Radiate enthusiasm, and let it be the wildfire that sparks change.', 'Radiate positivity, for it is the beacon that lights your way.', 'Radiate grace, and let it be the music of your soul.', 'Radiate gratitude, for it turns even the smallest gifts into treasures.', 'Radiate joy, and let it be the echo of your heart.', 'Radiate love, for it is the language that transcends all barriers.', 'Radiate confidence, and let it be the armor that shields your heart.', 'Radiate authenticity, for it is the mark of a truly genuine soul.', 'Radiate peace, and let it be the refuge for weary hearts.', 'Radiate resilience, and let it be the bedrock of your strength.', 'Radiate wisdom, for it carries the echoes of generations.', 'Radiate forgiveness, and let it be the key that unlocks your heart.', 'Radiate compassion, for it is the truest expression of your humanity.', 'Radiate strength, and let it be the fortress of your soul.', 'Radiate hope, and let it be the light that guides your way.', 'Radiate humility, and let it be the tapestry of your character.', 'Radiate determination, and let it be the force that drives you forward.', 'Radiate enthusiasm, and let it be the spark that ignites transformation.', 'Radiate positivity, for it is the energy that propels you forward.', 'Radiate grace, and let it be the fragrance of your presence.', 'Radiate gratitude, for it turns every moment into a gift.', 'Radiate kindness, and become a force for good in the world.', "Radiate joy, and let it be the chorus of your life's song.", 'Radiate love, for it is the essence of your being.', 'Radiate confidence, and let it be the foundation of your success.', 'Radiate authenticity, for it is the truest expression of self.', 'Radiate peace, and create a haven of serenity around you.', 'Radiate resilience, and let it be the beacon of your strength.', 'Radiate wisdom, for it carries the echoes of ages.', 'Radiate forgiveness, and let it be the bridge that mends hearts.', 'Radiate compassion, for it is the heartbeat of your soul.', 'Radiate strength, and let it be the lighthouse in the storms of life.', 'Radiate hope, and let it be the sunrise in your darkest moments.', 'Radiate curiosity, for it is the compass of endless discovery.', 'Radiate determination, and let it be the fuel that powers your journey.', 'Radiate enthusiasm, and let it be the spark that ignites change.', 'Radiate positivity, for it is a beacon that lights the way for others.', 'Believe in yourself, for you are capable of greatness.', 'Believe in yourself, and you will be unstoppable.', 'Believe in yourself, and the world will believe in you.', 'Believe in yourself, and you will conquer any challenge.', 'Believe in yourself, for you hold the power within.', 'Believe in yourself, and your dreams will take flight.', 'Believe in yourself, and let your light shine bright.', 'Believe in yourself, and miracles will happen.', 'Believe in yourself, and you will find your way.', 'Believe in yourself, and magic will follow.', 'Believe in yourself, and you will achieve the impossible.', 'Believe in yourself, for you are stronger than you think.', 'Believe in yourself, and watch your dreams unfold.', 'Believe in yourself, and you will inspire others.', 'Believe in yourself, and let your spirit soar.', 'Believe in yourself, for you are enough just as you are.', 'Believe in yourself, and you will overcome any obstacle.', 'Believe in yourself, and the world becomes your canvas.', 'Believe in yourself, and you will create your destiny.', 'Believe in yourself, and you will make a difference.', 'Believe in yourself, and you will leave a legacy.', 'Believe in yourself, and you will find the way forward.', 'Believe in yourself, and the universe will conspire in your favor.', 'Believe in yourself, and you will radiate confidence.', 'Believe in yourself, for you are worthy of every dream.', 'Believe in yourself, and you will touch the stars.', 'Believe in yourself, and watch the world transform around you.', 'Believe in yourself, and you will find your true purpose.', 'Believe in yourself, and let your heart lead the way.', 'Believe in yourself, for you are a masterpiece in the making.', 'Believe in yourself, and you will leave footprints of inspiration.', 'Believe in yourself, and you will be a guiding light for others.', 'Believe in yourself, and you will break through any barrier.', 'Believe in yourself, and you will forge a path to success.', 'Believe in yourself, and you will be a beacon of hope.', 'Believe in yourself, for your potential is limitless.', 'Believe in yourself, and you will inspire others to believe too.', 'Believe in yourself, and you will be a force of nature.', 'Believe in yourself, and you will unlock the doors of possibility.', 'Believe in yourself, for you are destined for greatness.', 'Believe in yourself, and you will find the strength to persevere.', 'Believe in yourself, and you will leave a mark on the world.', 'Believe in yourself, and you will be a light in the darkness.', 'Believe in yourself, and you will create your own destiny.', 'Believe in yourself, and you will be the change you seek.', 'Believe in yourself, and you will be a beacon of light for others.', 'Believe in yourself, and you will overcome any adversity.', 'Believe in yourself, for you have the power to transform.', 'Believe in yourself, and you will be a source of inspiration.', 'Believe in yourself, and you will achieve greatness.', 'Believe in yourself, and you will touch the lives of many.', 'Believe in yourself, for you are capable of extraordinary things.', 'Believe in yourself, and you will leave a legacy of love.', 'Believe in yourself, and you will discover your true potential.', 'Believe in yourself, and you will be a source of strength for others.', 'Believe in yourself, for you are a work of art in progress.', 'Believe in yourself, and you will find the path to success.', 'Believe in yourself, and you will light up the world.', 'Believe in yourself, for you have the power to create change.', 'Believe in yourself, and you will be a guiding star for others.', 'Believe in yourself, and you will overcome any challenge.', 'Believe in yourself, and you will be a beacon of joy.', 'Believe in yourself, and you will be an agent of transformation.', 'Believe in yourself, and you will be a force for good.', 'Believe in yourself, for you are capable of amazing things.', 'Believe in yourself, and you will be a source of light for others.', 'Believe in yourself, and you will break through any limitation.', 'Believe in yourself, and you will create a world of possibilities.', 'Believe in yourself, for you have the power to change lives.', 'Believe in yourself, and you will touch the hearts of many.', 'Believe in yourself, for you have the potential to soar.', 'Believe in yourself, and you will be a force for positivity.', 'Believe in yourself, and you will find the way to success.', 'Believe in yourself, and you will be an agent of change.', 'Believe in yourself, for you have the power to create magic.', 'Believe in yourself, and you will break through any obstacle.', 'Believe in yourself, for you have the potential to make a difference.', 'Believe in yourself, and you will be a force for positive change.', 'Believe in yourself, for you have the power to inspire.', 'Believe in yourself, and you will be a beacon of love.', 'Believe in yourself, for you are capable of achieving anything.', 'Believe in yourself, and you will be a beacon of light.', 'Believe in yourself, for you are the architect of your destiny.', "Life's journey is a canvas; paint it with vibrant experiences.", "Life's challenges are stepping stones to wisdom and growth.", "Life's melodies are composed of moments that touch the soul.", "Life's colors are brightest when painted with love and kindness.", "Life's adventures are the chapters that make our story extraordinary.", "Life's storms are followed by rainbows of hope and resilience.", "Life's garden blooms with the seeds of kindness we plant.", "Life's music is the symphony of laughter and joy.", "Life's whispers hold the secrets of the universe within them.", "Life's dance is more beautiful when we move with grace and gratitude.", "Life's tapestry is woven from threads of experience and learning.", "Life's compass points towards our true purpose and passion.", "Life's sunrise brings a new day filled with endless possibilities.", "Life's symphony is the harmonious blend of our dreams and reality.", "Life's path is illuminated by the light of our inner wisdom.", "Life's beauty is reflected in the eyes of those we love.", "Life's blessings multiply when we share them with others.", "Life's adventure is the pursuit of our heart's deepest desires.", "Life's rain nourishes the seeds of our dreams and aspirations.", "Life's canvas is painted with the strokes of our intentions and actions.", "Life's rhythm is the heartbeat of our existence; embrace it.", "Life's mosaic is made of the pieces of our unique experiences.", "Life's symphony is composed of the moments that take our breath away.", "Life's journey is the classroom where we learn to love unconditionally.", "Life's book is written with the ink of our thoughts and deeds.", "Life's garden flourishes when nurtured with care and gratitude.", "Life's masterpiece is created by the choices we make every day.", "Life's whispers guide us towards our true calling and purpose.", "Life's dance is a celebration of every step we take forward.", "Life's journey is a tapestry woven with threads of love.", "Life's challenges are opportunities in disguise; embrace them.", "Life's chapters unfold to reveal the story of our resilience.", "Life's colors are brighter when we spread kindness and joy.", "Life's canvas is painted with the strokes of our dreams and actions.", "Life's symphony is the melody of our heart's deepest desires.", "Life's garden blooms with the seeds of our intentions and dreams.", "Life's blessings are the gifts we receive and give in return.", "Life's adventure is the pursuit of our passions and dreams.", "Life's rain washes away the old and makes way for the new.", "Life's mosaic is a reflection of the beauty in our diversity.", "Life's symphony is composed of the moments that define us.", "Life's journey is a classroom where we learn to love and forgive.", "Life's book is written with the pen of our experiences and choices.", "Life's garden flourishes when tended with love and care.", "Life's masterpiece is created by the love we share with others.", "Life's whispers are the nudges from our inner wisdom.", "Life's dance is a celebration of the beauty in every step we take.", "Life's journey is a tapestry woven with threads of resilience.", "Life's challenges are opportunities to grow and become stronger.", "Life's chapters unfold to reveal the triumphs of our spirit.", "Life's colors are more vibrant when we choose to spread joy.", "Life's canvas is painted with the hues of our dreams and aspirations.", "Life's symphony is the music that plays in our hearts.", "Life's garden flourishes when watered with gratitude and kindness.", "Life's blessings are the treasures found in everyday moments.", "Life's adventure is the pursuit of purpose and meaning.", "Life's rain cleanses the soul and renews our sense of purpose.", "Life's mosaic is a testament to the beauty of our uniqueness.", "Life's symphony is composed of the moments that touch our hearts.", "Life's journey is a classroom where we learn to love ourselves.", "Life's book is written with the stories of our experiences.", "Life's garden flourishes when tended with patience and compassion.", "Life's masterpiece is created by the love we give and receive.", "Life's whispers are the guidance from our inner compass.", "Life's dance is a celebration of the journey we've traveled.", "Life's journey is a tapestry woven with threads of love and laughter.", "Life's challenges are opportunities to discover our inner strength.", "Life's chapters unfold to reveal the beauty of our resilience.", "Life's colors are more vibrant when painted with kindness.", "Life's canvas is painted with the strokes of our hopes and dreams.", "Life's symphony is the music that resonates in our hearts.", "Life's garden flourishes when nurtured with acts of kindness.", "Life's blessings are the moments of joy we share with others.", "Life's rain cleanses the soul and brings forth new beginnings.", "Life's mosaic is a reflection of the beauty in our differences.", "Life's journey is a classroom where we learn to love and let go.", "Life's book is written with the chapters of our experiences.", "Life's garden flourishes when watered with acts of love.", "Life's masterpiece is created by the love we offer to the world.", "Life's whispers are the reminders of our inner wisdom.", "Life's dance is a celebration of every step we take towards our dreams.", "Life's journey is a tapestry woven with threads of love and hope.", "Life's challenges are opportunities to discover our true potential.", "Life's chapters unfold to reveal the strength of our spirit.", "Life's colors are more vibrant when painted with empathy.", "Life's canvas is painted with the strokes of our aspirations and actions.", "Life's garden flourishes when tended with compassion and understanding.", "Life's blessings are the moments of joy that light up our days.", "Life's rain cleanses the soul and nourishes our inner gardens.", "Life's mosaic is a testament to the beauty of our diversity.", "Life's symphony is composed of the moments that resonate in our souls.", "Life's journey is a classroom where we learn to cherish every moment.", "Life's book is written with the stories of our journey and growth.", "Life's garden flourishes when tended with acts of kindness and love.", "Life's whispers are the reminders of our inner strength and wisdom.", "Dance through life's challenges with grace and determination.", 'Dance through each day with a heart full of gratitude.', 'Dance through adversity, for it is the rhythm of resilience.', 'Dance through the storms, for they will pass, leaving you stronger.', 'Dance through uncertainty, for it is the prelude to new beginnings.', 'Dance through each moment, savoring the beauty it holds.', 'Dance through the garden of life, sowing seeds of kindness.', 'Dance through the seasons, embracing the change they bring.', 'Dance through your fears, for on the other side lies freedom.', 'Dance through the rhythm of your heart; it knows the way.', 'Dance through the melodies of laughter; they are the sweetest.', 'Dance through the chapters of your story, embracing every twist.', 'Dance through the symphony of love, let your heart lead the way.', 'Dance through the colors of joy, painting your world with brightness.', 'Dance through the tapestry of life, leaving threads of inspiration.', 'Dance through the moments, for they are the notes of your song.', "Dance through life's garden, tending to the blooms of kindness.", 'Dance through the challenges, they are the steps to your growth.', 'Dance through the rain, for it cleanses and renews your spirit.', 'Dance through the canvas of your dreams, let your heart paint the way.', 'Dance through the echoes of your soul, they carry your true essence.', 'Dance through the rhythm of possibility; it knows no limits.', 'Dance through the symphony of gratitude, for it will lift you up.', 'Dance through the colors of authenticity, let your true self shine.', "Dance through life's tapestry, for it is woven with purpose.", 'Dance through the whispers of intuition; they are your inner compass.', 'Dance through the chapters of your journey, each one holds its magic.', 'Dance through the symphony of connection; it weaves hearts together.', 'Dance through the garden of kindness, sowing seeds of compassion.', 'Dance through the storms, for they are the birthplace of strength.', 'Dance through the pages of your story, write it with love and courage.', 'Dance through the colors of possibility, for they hold your dreams.', 'Dance through the symphony of hope, let it be your guiding star.', 'Dance through the rhythm of purpose; let your heart lead the way.', "Dance through life's garden, cultivating blooms of love and joy.", 'Dance through the chapters of growth, for they shape your journey.', 'Dance through the rain, for it brings life to the seeds of your dreams.', 'Dance through the canvas of your destiny, paint it with intention.', 'Dance through the echoes of your heart, they hold the truth of you.', 'Dance through the melody of change, for it is the heartbeat of life.', 'Dance through the symphony of gratitude, and watch your world transform.', 'Dance through the colors of self-discovery, let your soul be your guide.', "Dance through life's tapestry, and weave it with threads of love.", 'Dance through the whispers of your dreams; they hold the keys to your destiny.', 'Dance through the chapters of growth, and let your spirit soar.', 'Dance through the symphony of connection, for it is the heartbeat of humanity.', 'Dance through the garden of compassion, let kindness be your legacy.', 'Dance through the storms, for they cleanse and renew your spirit.', 'Dance through the pages of your story, and let your heart write the way.', 'Dance through the colors of possibility, for they hold the hues of your dreams.', 'Dance through the symphony of hope, and let it be your guiding light.', "Dance through life's garden, for it is rich with the fruits of love and joy.", 'Dance through the chapters of growth, for they are the pillars of your journey.', 'Dance through the rain, for it waters the seeds of your aspirations.', 'Dance through the canvas of your destiny, paint it with intention and love.', 'Dance through the echoes of your heart, for they resonate with your true self.', 'Dance through the melody of change, for it is the heartbeat of transformation.', 'Dance through the symphony of gratitude, and watch how it transforms your world.', 'Dance through the colors of self-discovery, for they unveil the brilliance of you.', "Dance through life's tapestry, weaving it with threads of love and joy.", 'Dance through the garden of compassion, letting kindness be your legacy.', 'Dance through the pages of your story, letting your heart write the way.', 'Dance through the canvas of your destiny, painting it with intention and love.', 'Let your heart lead the way, for it knows the path to true happiness.', 'Let your dreams be the wings that carry you to new heights.', "Let your kindness be the light that brightens someone's day.", 'Let your courage be stronger than your fears, and you will conquer anything.', "Let your actions speak louder than words, and you'll inspire those around you.", 'Let your passions be the compass that guides you to your purpose.', 'Let your laughter echo through the halls of your life.', 'Let your love be the force that transforms the world around you.', 'Let your creativity flow, for it is the source of endless possibilities.', 'Let your gratitude be the foundation of a joyful heart.', 'Let your authenticity shine, for it is the key to true connection.', 'Let your light shine so brightly that others find their way by it.', 'Let your voice be the anthem of your truth.', "Let your inner strength be your armor against life's challenges.", 'Let your curiosity be the spark that ignites your passions.', 'Let your wisdom be the North Star that guides your journey.', 'Let your empathy be the bridge that connects hearts.', 'Let your confidence be the wind in your sails on the sea of life.', 'Let your resilience be the rock on which you stand tall.', 'Let your generosity be the ripple that creates waves of kindness.', 'Let your intuition be the compass that leads you home.', 'Let your dreams be the driving force behind your actions.', 'Let your kindness be the language that transcends all barriers.', 'Let your determination be the fire that fuels your success.', 'Let your imagination be the canvas for your wildest dreams.', 'Let your heart be open to the beauty and wonder of the world.', 'Let your authenticity be the mirror that reflects your true self.', 'Let your love be the force that heals and binds.', 'Let your actions be a testament to your character.', 'Let your passions be the fuel that propels you forward.', 'Let your laughter be infectious, spreading joy wherever you go.', 'Let your voice be the melody that soothes and uplifts.', 'Let your inner light shine, for it is the beacon of your soul.', 'Let your courage be the sword that cuts through fear.', 'Let your gratitude be the compass that guides you home.', 'Let your authenticity be the key to unlocking genuine connections.', 'Let your love be the foundation of every action and decision.', 'Let your creativity be the wellspring of innovation and change.', 'Let your kindness be the balm that soothes wounded hearts.', 'Let your wisdom be the lighthouse in the stormy seas of life.', 'Let your empathy be the bridge that fosters understanding.', 'Let your confidence be the wind beneath your wings.', 'Let your resilience be the armor that shields your heart.', 'Let your generosity be the legacy you leave behind.', 'Let your intuition be the guiding star of your journey.', 'Let your dreams be the driving force behind your achievements.', 'Let your kindness be the currency of your interactions.', 'Let your determination be the fire that fuels your progress.', 'Let your imagination be the canvas for your aspirations.', 'Let your heart be open to the wonders of the world.', 'Let your authenticity be the mirror that reflects your soul.', 'Let your love be the force that heals and unites.', 'Let your actions be the echo of your principles.', 'Let your passions be the engine that drives your purpose.', 'Let your laughter be the contagion that spreads joy.', 'Let your voice be the symphony that resonates in the hearts of others.', 'Let your inner light shine, for it is the compass of your soul.', 'Let your courage be the sword that cuts through doubt.', 'Let your gratitude be the North Star that guides you home.', 'Let your authenticity be the bridge that connects hearts.', 'Let your creativity be the source of innovation and transformation.', 'Let your kindness be the salve that soothes wounded hearts.', 'Let your wisdom be the lighthouse in the turbulent seas of life.', 'Let your confidence be the wind that propels you forward.', 'Let your resilience be the armor that protects your heart.', 'Let your intuition be the compass that guides your way.', 'Let your dreams be the fuel that powers your journey.', 'Let your actions be the embodiment of your values.', 'Let your passions be the driving force behind your purpose.', 'Let your laughter be the infectious melody of joy.', 'Every day is a fresh canvas; paint it with vibrant strokes of kindness.', 'Every step forward is a victory on the path to your dreams.', "Every smile shared is a beam of light in someone's day.", 'Every challenge faced is an opportunity to grow stronger.', 'Every moment of gratitude is a step towards a joyful heart.', 'Every act of kindness ripples through the world, creating waves of goodness.', 'Every dream nurtured has the power to transform into reality.', 'Every heartbeat is a reminder of the preciousness of life.', 'Every sunrise brings new hope and the promise of a fresh start.', 'Every act of forgiveness frees the soul from its burdens.', 'Every word spoken has the potential to inspire and uplift.', 'Every effort counts, for it is the journey that shapes us.', 'Every friendship formed is a thread in the tapestry of life.', 'Every act of love is a testament to the beauty of the human spirit.', 'Every choice made is a brushstroke on the canvas of destiny.', 'Every challenge overcome is a testament to inner strength.', 'Every act of generosity creates a world of abundance.', 'Every thought has the power to shape your reality.', 'Every setback is a setup for a comeback.', 'Every experience, good or bad, is a lesson to be embraced.', 'Every sunset is a reminder of the beauty in letting go.', 'Every act of empathy is a bridge between hearts.', 'Every breath is a reminder of the gift of life.', 'Every act of gratitude is a step towards a contented heart.', 'Every dream nurtured has the power to bloom into reality.', 'Every kindness shown has the potential to change a life.', 'Every step taken in faith is a step towards miracles.', 'Every challenge faced is an opportunity for growth and learning.', 'Every person you meet has a story that deserves compassion.', 'Every moment of laughter is a symphony of joy in the heart.', 'Every act of forgiveness is a liberation of the soul.', 'Every word of encouragement is a beacon of hope.', 'Every effort towards your dreams is an investment in your future.', 'Every friendship formed is a treasure in the chest of memories.', 'Every act of love is a tribute to the power of the heart.', 'Every choice made is a brushstroke on the canvas of your destiny.', 'Every challenge overcome is a testament to your inner strength.', 'Every act of generosity creates a ripple of kindness.', 'Every thought holds the power to shape your reality.', 'Every experience, whether joyful or painful, is a chapter in your story.', 'Every sunset reminds us of the beauty in transitions.', 'Every act of empathy is a connection forged between souls.', 'Every breath is a gift, a reminder of the miracle of life.', 'Every act of gratitude is a step towards inner peace.', 'Every dream nurtured holds the promise of becoming reality.', 'Every act of kindness holds the potential to change a life.', 'Every challenge faced is an opportunity for growth and wisdom.', 'Every person you encounter carries a story worthy of compassion.', 'Every moment of laughter is a chorus of joy in the heart.', 'Every act of forgiveness is a liberation of the spirit.', 'Every word of encouragement is a beacon of light and strength.', 'Every effort directed towards your dreams is an investment in your future.', 'Every friendship formed is a gem in the treasury of memories.', 'Every act of love is a tribute to the boundless power of the heart.', 'Every choice made is a brushstroke on the canvas of your fate.', 'Every challenge overcome is a testament to your indomitable spirit.', 'Every act of generosity creates a wave of kindness and abundance.', 'Every thought has the potential to mold your reality.', 'Every setback is a setup for a glorious comeback.', 'Every experience, whether joyful or painful, is a chapter in your narrative.', 'Every sunset reminds us of the beauty in transitions and farewells.', 'Every act of empathy is a bridge that connects souls.', 'Every breath is a gift, a reminder of the marvel of existence.', 'Every act of gratitude is a step towards the tranquility of the heart.', 'Every dream nurtured holds the promise of blossoming into reality.', "Every act of kindness holds the potential to alter a life's course.", 'Every step taken in faith is a step towards witnessing miracles.', 'Every challenge faced is an opportunity for growth and enlightenment.', 'Every person you encounter carries a story worthy of your empathy.', 'Every moment of laughter is a symphony of joy that reverberates in the heart.', 'Every act of forgiveness is a liberation of the soul from bitterness.', 'Every word of encouragement is a beacon of light and a wellspring of strength.', 'Every effort directed towards your dreams is an investment in a brighter future.', 'Every friendship formed is a gem, a treasure in the treasury of cherished memories.', 'Every act of love is a tribute to the boundless power of the heart to heal and transform.', 'Every choice made is a brushstroke, a stroke of the brush on the canvas of your destiny.', 'Every challenge overcome is a testament, a testament to your indomitable spirit and strength.', 'Every act of generosity creates a wave, a wave of kindness that ripples out, creating abundance.', 'Every thought holds the potential, the potential to shape your reality and influence your world.', 'Every setback is a setup, a setup for a triumphant comeback, a return more powerful and determined.', 'Every experience, whether joyous or painful, is a chapter, a chapter in the story of your journey, a tale worth telling.', 'Every sunset reminds us, reminds us of the beauty, the beauty in transitions, in the ebb and flow of life, and the gentle goodbyes.', 'Every act of empathy is a bridge, a bridge that connects souls, transcending differences, and fostering understanding and compassion.', 'Every breath is a gift, a gift from the universe, a reminder of the marvel, the marvel of existence and the boundless possibilities it brings.', 'Every act of gratitude is a step, a step towards the tranquility, the tranquility of the heart, a serene state where joy and contentment reside.', 'Every dream nurtured holds the promise, the promise of blossoming, of blooming into reality, of becoming a vibrant and significant part of your life.', "Every act of kindness holds the potential, the potential to alter, to change a life's course, to be the turning point in someone's journey, a beacon of hope.", 'Every step taken in faith is a step, a step towards witnessing, witnessing miracles, the extraordinary moments that defy logic and touch the soul.', 'Every challenge faced is an opportunity, an opportunity for growth, for learning, for evolving into a stronger, wiser, and more resilient version of yourself.', 'Every person you encounter carries a story, a story worthy, worthy of your empathy, of your understanding, for within every story lies a universe of experiences and emotions.', 'Every moment of laughter is a symphony, a symphony of joy, a melody that resonates, that reverberates in the heart, spreading warmth and happiness far and wide.', 'Every act of forgiveness is a liberation, a liberation of the soul, a release from the chains of bitterness and resentment, a path towards healing and inner peace.', 'Every word of encouragement is a beacon, a beacon of light, a guiding star that illuminates the path, providing strength, hope, and the assurance that you are not alone.', 'Every effort directed towards your dreams is an investment, an investment in a brighter future, in the realization of your aspirations, and in the fulfillment of your potential.', 'Every friendship formed is a gem, a gem in the treasury, the treasury of cherished memories, a testament to the beauty and richness that meaningful connections bring to life.', 'Every act of love is a tribute, a tribute to the boundless power, the boundless power of the heart, to heal and transform, to connect and uplift, to create a legacy of warmth and affection.', 'Every choice made is a brushstroke, a stroke of the brush, a deliberate mark on the canvas, the canvas of your destiny, each decision shaping the masterpiece that is your unique life story.', 'Every challenge overcome is a testament, a testament to your indomitable spirit, a living proof of your strength and resilience, a reminder that you are capable of conquering any obstacle that comes your way.', 'Singapore, where cultures converge, and dreams take flight.', 'In the heart of Singapore, innovation knows no bounds.', "From Marina Bay to Orchard Road, Singapore's beauty knows no bounds.", 'In Singapore, every sunrise brings new opportunities to shine.', 'The Lion City roars with ambition and embraces diversity.', 'Singapore, where traditions dance with modernity in perfect harmony.', "In Singapore's skyline, dreams reach for the stars.", "From Sentosa's beaches to Chinatown's alleys, Singapore's charm is boundless.", "In the heart of Southeast Asia, Singapore's pulse beats strong.", "Singapore's gardens bloom with beauty and possibility.", "From Raffles Place to Clarke Quay, Singapore's energy is contagious.", 'In Singapore, every corner holds a tale waiting to be told.', "The Merlion stands tall, a symbol of Singapore's strength and grace.", "From hawker centers to fine dining, Singapore's flavors are diverse and delectable.", 'In the Lion City, innovation sparks and ideas take flight.', "From Little India's colors to the Botanic Gardens' serenity, Singapore's diversity is its strength.", "Singapore's skyline glistens with aspirations and dreams.", 'In the heart of Singapore, every moment is a chance for adventure.', "From Gardens by the Bay to Universal Studios, Singapore's attractions are world-class.", 'Singapore, where dreams are nurtured and futures are built.', 'In the Lion City, each day is a canvas for new beginnings.', "From Merlion Park to Bukit Timah Nature Reserve, Singapore's beauty is boundless.", "Singapore's skyline reaches for the heavens, a testament to its ambition.", 'In the heart of this island nation, resilience defines its spirit.', "From Orchard Road's shopping to Chinatown's history, Singapore's tapestry is rich and vibrant.", 'Singapore, where cultures intertwine, creating a tapestry of unity.', 'In Singapore, each day brings new opportunities to shine.', "From Sentosa's shores to Pulau Ubin's trails, Singapore's nature is a treasure.", "Singapore's skyline tells a story of progress and vision.", 'In the heart of this Lion City, every step is a leap of faith.', "From Marina Bay Sands to the Singapore Flyer, the Lion City's landmarks stand tall.", 'Singapore, where resilience shapes the character of a nation.', 'In Singapore, every dawn carries the promise of a brighter tomorrow.', "From Chinatown's lanterns to Kampong Glam's colors, Singapore's heritage is alive.", "Singapore's skyline illuminates the night with dreams and aspirations.", 'In the heart of this vibrant city-state, dreams find their wings.', "From Esplanade's performances to Haw Par Villa's tales, Singapore's culture is alive and thriving.", 'Singapore, where determination fuels progress and innovation.', 'In Singapore, every sunset paints the sky with wonder.', "From Pulau Tekong's quiet to the Southern Islands' serenity, Singapore's tranquility is a treasure.", "Singapore's skyline testifies to the nation's ambition and vision.", 'In the heart of this Lion City, hope lights the way.', "From Fort Canning's history to Labrador Park's ruggedness, Singapore's past is cherished.", 'Singapore, where passion and dedication shape the future.', 'In Singapore, every opportunity is a stepping stone to success.', "From Jurong Bird Park's colors to Night Safari's mysteries, Singapore's wildlife is a wonder.", "Singapore's skyline stretches towards the horizon, a symbol of endless potential.", 'In the heart of this Lion City, love and unity reign.', "From East Coast Park's breezes to Pulau Semakau's serenity, Singapore's nature is a sanctuary.", 'Singapore, where dreams are nurtured and talents are honed.', 'In Singapore, every endeavor is a step towards excellence.', "From Peranakan houses to colonial architecture, Singapore's history is preserved with pride.", "Singapore's skyline stands tall, a beacon of progress and innovation.", 'In the heart of this Lion City, passion ignites change.', "From Changi Airport's efficiency to Singapore River's vibrancy, Singapore's energy is palpable.", 'Singapore, where unity and diversity create a harmonious symphony.', 'In Singapore, every sunrise is a canvas for new beginnings.', "From MacRitchie Reservoir's calm to Sungei Buloh Wetland Reserve's vitality, Singapore's nature is a sanctuary.", "Singapore's skyline paints a picture of a nation reaching for the stars.", 'In the heart of this Lion City, dreams find their wings and take flight.', "From Singapore Zoo's wonders to River Safari's adventures, Singapore's wildlife is a treasure.", 'Singapore, where diligence and ingenuity shape the future.', "In Singapore, every sunset whispers promises of tomorrow's beauty.", "From Kent Ridge Park's views to Pulau Ubin's simplicity, Singapore's nature is a haven.", "Singapore's skyline embodies the spirit of progress and determination.", 'In the heart of this Lion City, compassion lights the way.', "From Changi Village's tranquility to Punggol Waterway's vibrancy, Singapore's landscapes are diverse and breathtaking.", 'Singapore, where heritage and modernity dance in perfect harmony.', 'In Singapore, every dawn brings the hope of a brand new day.', "From Labrador Nature Reserve's rugged beauty to Jurong Lake Gardens' serenity, Singapore's nature is a sanctuary.", "Singapore's skyline stands tall, a testament to a nation's aspirations.", 'In the heart of this Lion City, kindness defines the community.', "From Kranji Countryside's tranquility to Southern Ridges' breathtaking views, Singapore's nature is diverse and breathtaking.", 'Singapore, where determination and perseverance shape destinies.', 'In Singapore, every opportunity is a gateway to a brighter future.', "From MacRitchie Reservoir's tranquility to Chek Jawa's biodiversity, Singapore's nature is a marvel.", "Singapore's skyline tells a story of progress, resilience, and unwavering vision.", 'In the heart of this Lion City, hope blooms like a garden in full bloom.', "From Bukit Batok Nature Park's tranquility to Pulau Hantu's marine wonder, Singapore's nature is a treasure.", 'Singapore, where unity and diversity intertwine, creating a vibrant tapestry.', 'In Singapore, every sunrise paints the sky with hues of possibility.', "From Labrador Nature Reserve's rugged charm to Bukit Timah Nature Reserve's grandeur, Singapore's nature is a sanctuary.", "Singapore's skyline paints a picture of a nation that never stops reaching for the stars.", 'In the heart of this Lion City, dreams are nurtured and realized.', "From Sungei Buloh Wetland Reserve's biodiversity to Bukit Brown Cemetery's heritage, Singapore's nature is a treasure trove.", 'Singapore, where diligence and determination shape destinies.', 'In Singapore, every endeavor is a step towards excellence and progress.', "From Coney Island's simplicity to Pulau Ubin's rugged charm, Singapore's nature is a sanctuary.", "Singapore's skyline embodies the spirit of progress and the vision of tomorrow.", 'In the heart of this Lion City, compassion and empathy flourish.', "From Labrador Nature Reserve's tranquility to Hindhede Nature Park's serenity, Singapore's nature is diverse and breathtaking.", 'Singapore, where heritage and modernity dance in perfect harmony, creating a unique symphony.', 'In Singapore, every dawn carries the promise of new beginnings and fresh opportunities.', "From Chestnut Nature Park's tranquility to Bukit Batok Nature Park's charm, Singapore's nature is a sanctuary.", "Singapore's skyline stands tall, a testament to the nation's resilience and unwavering determination.", 'In the heart of this Lion City, kindness and generosity shape the community.', "From Pulau Ubin's simplicity to Chek Jawa's biodiversity, Singapore's nature is a marvel.", 'Singapore, where determination and innovation pave the way for a brighter future.', 'In Singapore, every opportunity is a door to a world of potential and possibility.', "From Sungei Buloh Wetland Reserve's biodiversity to Jurong Lake Gardens' beauty, Singapore's nature is a treasure trove.", 'Our planet, a precious jewel in the cosmos, deserves our utmost care.', 'In the embrace of nature, we find the true heartbeat of our planet.', "Our planet's beauty is a reflection of the wonders of the universe.", 'Every sunrise is a reminder of the hope and potential within our planet.', 'Through unity and stewardship, we can protect the legacy of our planet.', "Our planet's landscapes are a tapestry woven with threads of life.", 'In the whispers of the wind, we hear the soulful song of our planet.', 'Every tree is a testament to the strength and resilience of our planet.', "Our planet's oceans hold secrets and mysteries waiting to be explored.", 'In the dance of seasons, we witness the ever-changing face of our planet.', "Our planet's creatures, big and small, are a testament to diversity.", 'Every act of conservation is a love letter to the future of our planet.', "Our planet's forests are the lungs that breathe life into the Earth.", 'In the stillness of nature, we find the true essence of our planet.', "Our planet's rivers flow with the stories of civilizations past and present.", 'Every mountain peak reaches for the skies, echoing the spirit of our planet.', "Our planet's deserts hold the echoes of ancient tales and survival.", 'In the symphony of life, each species plays a vital role on our planet.', "Our planet's coral reefs are vibrant cities beneath the waves.", 'Every footprint we leave should be a mark of respect for our planet.', "Our planet's wetlands are sanctuaries for life to thrive and flourish.", 'In the eyes of a child, we see hope for the future of our planet.', "Our planet's glaciers are the ancient storytellers of time immemorial.", 'Every act of kindness towards nature is a gift to the soul of our planet.', "Our planet's grasslands are the cradle of biodiversity and resilience.", 'In the colors of a sunset, we glimpse the artistry of our planet.', "Our planet's tundras are a testament to life's adaptability and strength.", "Every drop of rain is a reminder of the Earth's life-giving embrace.", "Our planet's savannas are a canvas painted with the brushstrokes of survival.", 'In the flight of birds, we witness the freedom that our planet offers.', "Our planet's mangroves are the guardians of coastal life and stability.", 'Every act of conservation is a vote for the future of our planet.', "Our planet's canyons are the scars of time, etched with stories untold.", 'In the whispers of leaves, we hear the heartbeat of our planet.', "Our planet's volcanoes are the fiery heartbeat beneath the surface.", 'Every act of love towards nature is a pledge to protect our planet.', "Our planet's fjords are the silent witnesses of ancient ice ages.", "In the rustle of grass, we find the rhythm of our planet's breath.", "Our planet's caves hold secrets waiting to be uncovered and understood.", 'Every ray of sunlight is a promise of warmth and life for our planet.', "Our planet's deltas are the lifeblood of coastal communities and wildlife.", 'In the dance of fireflies, we see the magic that our planet holds.', "Our planet's plateaus are the stages for stories of endurance and adaptation.", "Every act of conservation is a legacy of care for our planet's future.", "Our planet's hot springs are the cauldrons of nature's alchemy and renewal.", "In the rustle of leaves, we hear the whispers of our planet's wisdom.", "Our planet's geysers are the reminders of the Earth's fiery core.", 'Every act of stewardship is a promise to safeguard the future of our planet.', "Our planet's lakes are the mirrors reflecting the skies and our aspirations.", 'In the flight of butterflies, we witness the transformation that our planet encourages.', "Our planet's meadows are the blankets that cradle diverse life forms.", 'Every seed planted is a gesture of hope for the future of our planet.', "Our planet's archipelagos are the jewels set in the crown of the sea.", "In the call of a bird, we hear the song of our planet's wild spirit.", "Our planet's atolls are the delicate rings that protect coastal ecosystems.", 'Every act of conservation is a promise to cherish and protect our planet.', "Our planet's estuaries are the nurseries of life for many coastal species.", "In the melody of a stream, we hear the symphony of our planet's vitality.", "Our planet's kelp forests are the underwater jungles of resilience.", 'Every footprint in the sand is a pledge to tread lightly on our planet.', "Our planet's caves are the silent galleries of nature's artistry.", "In the fragrance of a flower, we sense the perfume of our planet's beauty.", "Our planet's lagoons are the tranquil havens for myriad marine species.", "Every act of conservation is a beacon of hope for our planet's future.", "Our planet's oases are the life-giving heartbeats in arid landscapes.", "In the chorus of frogs, we hear the harmony of our planet's ecosystems.", "Our planet's oyster reefs are the fortresses of coastal protection.", "Every drop of dew is a testament to the resilience of our planet's lifeforms.", "Our planet's monsoons are the rhythmic heartbeat of the seasons.", "In the fragrance of a forest, we inhale the essence of our planet's vitality.", "Our planet's polar regions are the frigid frontiers of life's tenacity.", "Every act of conservation is a legacy of love for our planet's future.", "Our planet's orchards are the bountiful harvests of nature's abundance.", "In the hum of bees, we hear the industry of our planet's pollinators.", "Our planet's prairies are the symphonies of wind and grass, a testament to endurance.", 'Every leaf that falls is a reminder of the cycles of life on our planet.', "Our planet's rainforests are the lungs that breathe life into the atmosphere.", "Our planet's reefs are the vibrant cities beneath the waves, teeming with life.", 'Every act of conservation is a promise to protect the treasures of our planet.', "Our planet's rivers are the veins that carry life-giving waters across the land.", "In the whisper of leaves, we find the rhythm of our planet's breath.", "Our planet's sand dunes are the sculpted sculptures of nature's winds.", "Our planet's savannas are the theaters of survival, where life unfolds its drama.", "Our planet's seashores are the thresholds of life's transition from land to sea.", "Every act of conservation is a testament to our commitment to our planet's future.", "Our planet's snowflakes are the delicate brushstrokes of winter's artistry.", "In the rustle of leaves, we find the rhythm of our planet's breath.", "Our planet's soil is the cradle from which life springs forth.", "Our planet's springs are the wellsprings of life's refreshment and renewal.", "Our planet's steppes are the wide open canvases of nature's palette.", "Our planet's streams are the veins that pulse with life's vitality.", "Our planet's wetlands are the nurseries of biodiversity and life.", 'Every act of conservation is a promise to cherish and protect the future of our planet.', 'This morning holds the promise of a brand new day, full of possibilities.', 'In the gentle embrace of this morning, we find the magic of new beginnings.', 'This morning, let gratitude be your first thought, and joy will follow.', 'With each sunrise, this morning paints the sky with colors of hope.', "This morning, let kindness be the compass guiding your day's journey.", "In the stillness of this morning, hear the whispers of your heart's desires.", 'This morning, let your smile light up the world like the rising sun.', 'With every breath this morning, inhale peace, exhale gratitude.', 'This morning, let your actions be a testament to the goodness in your heart.', 'In the quietude of this morning, find solace in the beauty of simplicity.', 'This morning, let your heart beat in rhythm with the pulse of the universe.', 'With open arms, embrace the opportunities that this morning brings.', 'This morning, let your spirit dance to the melody of hope and possibility.', 'In the tender light of this morning, discover the beauty in every moment.', 'This morning, let gratitude be your guide and abundance will follow.', 'With each sunrise, this morning whispers, "Today is your canvas, paint it well."', 'This morning, let your actions be the seeds for a garden of kindness.', 'In the gentle embrace of this morning, find strength for the day ahead.', 'This morning, let the beauty of nature be your source of inspiration.', 'With a grateful heart, welcome the opportunities that this morning presents.', 'This morning, let your kindness be the ripple that touches every shore.', 'In the tranquility of this morning, discover the power of stillness.', 'This morning, let your laughter be the melody that fills the air.', 'With the dawn of this morning, let love be your guiding star.', "This morning, let the sun's warmth remind you of life's precious gifts.", 'In the serenity of this morning, find clarity in the stillness of your soul.', 'This morning, let your actions speak louder than any words ever could.', 'With each sunrise, this morning offers the promise of a fresh start.', 'This morning, let your heart overflow with gratitude for the gift of today.', 'In the embrace of this morning, find the strength to face any challenge.', 'This morning, let your presence be a gift to everyone you meet.', 'With an open heart, welcome the opportunities that this morning holds.', "This morning, let your kindness be the light that brightens someone's day.", 'In the quiet moments of this morning, find the wisdom of your soul.', 'This morning, let your smile be the beacon that brightens the world.', 'With the rising sun, let gratitude fill every corner of your heart this morning.', 'This morning, let your actions reflect the love that resides within you.', 'In the stillness of this morning, find the power of being present.', 'This morning, let your laughter echo through the chambers of your soul.', 'With each sunrise, this morning reminds us that every day is a gift.', 'This morning, let your kindness be the bridge that connects hearts.', 'In the serenity of this morning, find the courage to let go of what no longer serves you.', 'This morning, let your actions be the legacy of love you leave behind.', 'With open arms, embrace the potential that this morning offers.', "This morning, let gratitude be the soundtrack of your day's journey.", 'In the gentle embrace of this morning, find the energy to chase your dreams.', 'This morning, let your smile be the compass that guides you to joy.', 'With every breath, let gratitude fill your lungs and heart this morning.', 'This morning, let your actions be the embodiment of your truest self.', 'In the tranquility of this morning, find solace in the simple joys of life.', 'This morning, let your kindness be the legacy you leave in every heart.', 'With each sunrise, this morning paints a new canvas for your journey.', 'This morning, let your heart be the compass that leads you to happiness.', 'In the stillness of this morning, find the clarity to navigate your path.', 'This morning, let your actions be a symphony of compassion and love.', 'With the dawn of this morning, let gratitude illuminate your soul.', "This morning, let your presence be the gift that makes someone's day brighter.", 'In the embrace of this morning, find the strength to overcome any obstacle.', 'This morning, let your kindness be the spark that ignites positivity in the world.', 'With an open heart, welcome the opportunities that this morning brings your way.', 'This morning, let your laughter be the music that fills the air with joy.', 'In the serenity of this morning, find the power of being fully present.', 'This morning, let your actions be a testament to the goodness in your soul.', 'With each sunrise, this morning offers the promise of a fresh start and new beginnings.', 'In the embrace of this morning, find the courage to face the day with strength.', "This morning, let your kindness be the light that brightens someone's path.", 'With open arms, embrace the opportunities that this morning presents to you.', 'This morning, let your smile be the beacon that brightens the world around you.', 'In the quiet moments of this morning, find the wisdom that resides within your soul.', 'This morning, let your actions reflect the love that dwells within your heart.', 'This morning, let your kindness be the bridge that connects souls together.', 'In the stillness of this morning, find the power of being present in the moment.', 'This morning, let your laughter echo through the chambers of your heart and soul.', 'With each sunrise, this morning reminds us that every day is a precious gift.', 'This morning, let your kindness be the legacy you leave in every heart you touch.', 'This morning, let your actions be the legacy of love and compassion you leave behind.', 'With open arms, embrace the potential and possibilities that this morning offers.', "This morning, let gratitude be the soundtrack of your day's journey and adventures.", 'In the gentle embrace of this morning, find the energy to pursue your dreams and aspirations.', 'This morning, let your smile be the compass that guides you to moments of joy and happiness.', 'With every breath, let gratitude fill your lungs and heart, reminding you of the blessings of this morning.', 'This morning, let your actions be the embodiment of your truest and most beautiful self.', 'In the tranquility of this morning, find solace and peace in the simple joys of life.', 'This morning, let your kindness be the legacy you leave in every heart and soul you encounter.', 'With each sunrise, this morning paints a new canvas for your journey, a fresh start for new beginnings.', 'This morning, let your heart be the compass that leads you to moments of happiness and contentment.', 'In the stillness of this morning, find the clarity and focus to navigate your path with purpose and determination.', 'This morning, let your actions be a symphony of compassion, love, and kindness that resonates with the world.', 'With the dawn of this morning, let gratitude illuminate your soul, filling it with warmth and appreciation.', "This morning, let your presence be the gift that brightens someone's day, leaving a trail of smiles and warmth.", 'In the embrace of this morning, find the strength and fortitude to overcome any obstacle that comes your way.', 'This morning, let your kindness be the spark that ignites positivity and light in the hearts of others.', 'With an open heart, welcome the opportunities and possibilities that this morning brings your way.', 'This morning, let your laughter be the music that fills the air with joy and lightheartedness.', 'In the serenity of this morning, find the power and beauty of being fully present in the moment.', 'This morning, let your actions be a testament to the goodness, kindness, and compassion that reside within you.', 'With each sunrise, this morning offers the promise of a fresh start and new beginnings, reminding us of the preciousness of every day.']
# Demonstrate how to use the tokenizer
text_data = ["hello world my name is"]
tokenizer =Tokenizer()
tokenizer.fit_on_texts(text_data)
#Texts to sequences helps
sequences = tokenizer.texts_to_sequences(text_data)
# Create rolling window sequences
for window_size in range(1, len(text_data[0].split())-1):
rolling_sequences = [
sequences[0][i:i+window_size+1] for sequence in sequences
for i in range(len(sequence)-window_size)
]
print(f"Window Size of {window_size}: {rolling_sequences}")
Window Size of 1: [[1, 2], [2, 3], [3, 4], [4, 5]] Window Size of 2: [[1, 2, 3], [2, 3, 4], [3, 4, 5]] Window Size of 3: [[1, 2, 3, 4], [2, 3, 4, 5]]
# Perform it on the actual dataset
tokenizer = Tokenizer()
tokenizer.fit_on_texts(data_list)
all_rolling_sequences = []
for text in data_list:
sequences = tokenizer.texts_to_sequences([text])[0]
# Create rolling window sequences for different window size
rolling_sequence_list = []
for window_size in range(1, len(sequences)-1):
rolling_sequences = [
sequences[i:i+window_size+1] for i in range(len(sequences)-window_size)
]
rolling_sequence_list.append(rolling_sequences)
if text == data_list[0]:
print(f"Window Size of {window_size}: {rolling_sequences}")
# Combine rolling window sequences of different window size
all_rolling_sequences.extend([item for sublist in rolling_sequence_list for item in sublist])
Window Size of 1: [[17, 1], [1, 49], [49, 2], [2, 13], [13, 77], [77, 372], [372, 5], [5, 163], [163, 486], [486, 10], [10, 101], [101, 3], [3, 46], [46, 22], [22, 52]] Window Size of 2: [[17, 1, 49], [1, 49, 2], [49, 2, 13], [2, 13, 77], [13, 77, 372], [77, 372, 5], [372, 5, 163], [5, 163, 486], [163, 486, 10], [486, 10, 101], [10, 101, 3], [101, 3, 46], [3, 46, 22], [46, 22, 52]] Window Size of 3: [[17, 1, 49, 2], [1, 49, 2, 13], [49, 2, 13, 77], [2, 13, 77, 372], [13, 77, 372, 5], [77, 372, 5, 163], [372, 5, 163, 486], [5, 163, 486, 10], [163, 486, 10, 101], [486, 10, 101, 3], [10, 101, 3, 46], [101, 3, 46, 22], [3, 46, 22, 52]] Window Size of 4: [[17, 1, 49, 2, 13], [1, 49, 2, 13, 77], [49, 2, 13, 77, 372], [2, 13, 77, 372, 5], [13, 77, 372, 5, 163], [77, 372, 5, 163, 486], [372, 5, 163, 486, 10], [5, 163, 486, 10, 101], [163, 486, 10, 101, 3], [486, 10, 101, 3, 46], [10, 101, 3, 46, 22], [101, 3, 46, 22, 52]] Window Size of 5: [[17, 1, 49, 2, 13, 77], [1, 49, 2, 13, 77, 372], [49, 2, 13, 77, 372, 5], [2, 13, 77, 372, 5, 163], [13, 77, 372, 5, 163, 486], [77, 372, 5, 163, 486, 10], [372, 5, 163, 486, 10, 101], [5, 163, 486, 10, 101, 3], [163, 486, 10, 101, 3, 46], [486, 10, 101, 3, 46, 22], [10, 101, 3, 46, 22, 52]] Window Size of 6: [[17, 1, 49, 2, 13, 77, 372], [1, 49, 2, 13, 77, 372, 5], [49, 2, 13, 77, 372, 5, 163], [2, 13, 77, 372, 5, 163, 486], [13, 77, 372, 5, 163, 486, 10], [77, 372, 5, 163, 486, 10, 101], [372, 5, 163, 486, 10, 101, 3], [5, 163, 486, 10, 101, 3, 46], [163, 486, 10, 101, 3, 46, 22], [486, 10, 101, 3, 46, 22, 52]] Window Size of 7: [[17, 1, 49, 2, 13, 77, 372, 5], [1, 49, 2, 13, 77, 372, 5, 163], [49, 2, 13, 77, 372, 5, 163, 486], [2, 13, 77, 372, 5, 163, 486, 10], [13, 77, 372, 5, 163, 486, 10, 101], [77, 372, 5, 163, 486, 10, 101, 3], [372, 5, 163, 486, 10, 101, 3, 46], [5, 163, 486, 10, 101, 3, 46, 22], [163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 8: [[17, 1, 49, 2, 13, 77, 372, 5, 163], [1, 49, 2, 13, 77, 372, 5, 163, 486], [49, 2, 13, 77, 372, 5, 163, 486, 10], [2, 13, 77, 372, 5, 163, 486, 10, 101], [13, 77, 372, 5, 163, 486, 10, 101, 3], [77, 372, 5, 163, 486, 10, 101, 3, 46], [372, 5, 163, 486, 10, 101, 3, 46, 22], [5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 9: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10], [49, 2, 13, 77, 372, 5, 163, 486, 10, 101], [2, 13, 77, 372, 5, 163, 486, 10, 101, 3], [13, 77, 372, 5, 163, 486, 10, 101, 3, 46], [77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [372, 5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 10: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486, 10], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101], [49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3], [2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46], [13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [77, 372, 5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 11: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3], [49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46], [2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 12: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46], [49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 13: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22, 52]] Window Size of 14: [[17, 1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22], [1, 49, 2, 13, 77, 372, 5, 163, 486, 10, 101, 3, 46, 22, 52]]
# Pad the combined sequences
max_sequence_rolling_len = max([len(x) for x in all_rolling_sequences])
X_padded = pad_sequences(
[window[:-1] for window in all_rolling_sequences],
maxlen = max_sequence_rolling_len,
padding='pre'
)
y = [window[-1] for window in all_rolling_sequences]
y_categorical = to_categorical(y, num_classes=len(tokenizer.word_index)+1)
total_words_rolling = len(tokenizer.word_index) + 1 # index 0 is reserved for padding
print(f"Total number of words: {total_words_rolling}")
Total number of words: 1199
# Split the rolling window dataset into training, validation and test sets
X_train_roll, X_test_roll, y_train_roll, y_test_roll = train_test_split(
X_padded, y_categorical , test_size =0.2, random_state=42, shuffle=True
)
X_train_roll, X_val_roll, y_train_roll, y_val_roll = train_test_split(
X_train_roll, y_train_roll, test_size=0.25, random_state=42, shuffle=True
)
print(f"X_train: {X_train_roll.shape}")
print(f"y_train: {y_train_roll.shape}")
print(f"X_val: {X_val_roll.shape}")
print(f"y_val: {y_val_roll.shape}")
print(f"X_test: {X_test_roll.shape}")
print(f"y_test: {y_test_roll.shape}")
X_train: (36707, 34) y_train: (36707, 1199) X_val: (12236, 34) y_val: (12236, 1199) X_test: (12236, 34) y_test: (12236, 1199)
Models List:
# Plot accuracy_curve
def plot_learning_curve(history):
history_df = pd.DataFrame(history)
epochs = list(range(1,len(history_df)+1))
fig, ax = plt.subplots(1,2, figsize=(16,6))
# Training loss and validation loss
ax1=ax[0]
ax1.plot(epochs, history_df["loss"], label="Training Loss")
ax1.plot(epochs, history_df["val_loss"], label="Validation Loss")
ax1.legend()
ax1.set_ylabel("Loss")
ax1.set_xlabel("Number of Epochs")
ax1.set_title("Training and Validation Loss")
# Training accuracy and validation accuracy
ax2=ax[1]
ax2.plot(epochs, history_df["accuracy"], label="Training Accuracy")
ax2.plot(epochs, history_df["val_accuracy"], label="Validation Accuracy")
ax2.legend()
ax2.set_ylabel("Accuracy")
ax2.set_xlabel("Number of Epochs")
ax2.set_title("Training and Validation Accuracy")
plt.show()
Important Parameters:
tf.keras.backend.clear_session()
# Create the model
simpleRNN = Sequential(
name='simpleRNN_v1',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
SimpleRNN(64, activation='tanh'),
Dropout(0.4),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
simpleRNN.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
simpleRNN_history = simpleRNN.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 28s 23ms/step - loss: 5.2567 - accuracy: 0.1108 - val_loss: 4.7446 - val_accuracy: 0.1375 Epoch 2/100 1148/1148 [==============================] - 25s 22ms/step - loss: 4.4604 - accuracy: 0.1752 - val_loss: 4.1367 - val_accuracy: 0.2260 Epoch 3/100 1148/1148 [==============================] - 26s 22ms/step - loss: 3.8853 - accuracy: 0.2495 - val_loss: 3.6042 - val_accuracy: 0.3008 Epoch 4/100 1148/1148 [==============================] - 26s 22ms/step - loss: 3.5337 - accuracy: 0.3025 - val_loss: 3.2565 - val_accuracy: 0.3449 Epoch 5/100 1148/1148 [==============================] - 25s 22ms/step - loss: 3.1463 - accuracy: 0.3547 - val_loss: 2.9639 - val_accuracy: 0.3926 Epoch 6/100 1148/1148 [==============================] - 25s 22ms/step - loss: 2.9029 - accuracy: 0.3894 - val_loss: 2.7564 - val_accuracy: 0.4290 Epoch 7/100 1148/1148 [==============================] - 25s 22ms/step - loss: 2.7080 - accuracy: 0.4186 - val_loss: 2.5762 - val_accuracy: 0.4551 Epoch 8/100 1148/1148 [==============================] - 25s 22ms/step - loss: 2.5600 - accuracy: 0.4476 - val_loss: 2.4316 - val_accuracy: 0.4893 Epoch 9/100 1148/1148 [==============================] - 25s 22ms/step - loss: 2.4321 - accuracy: 0.4672 - val_loss: 2.3167 - val_accuracy: 0.5082 Epoch 10/100 1148/1148 [==============================] - 25s 21ms/step - loss: 2.3330 - accuracy: 0.4839 - val_loss: 2.2318 - val_accuracy: 0.5164 Epoch 11/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.2477 - accuracy: 0.4994 - val_loss: 2.1511 - val_accuracy: 0.5379 Epoch 12/100 1148/1148 [==============================] - 25s 22ms/step - loss: 2.1740 - accuracy: 0.5097 - val_loss: 2.0894 - val_accuracy: 0.5476 Epoch 13/100 1148/1148 [==============================] - 24s 21ms/step - loss: 2.1231 - accuracy: 0.5190 - val_loss: 2.0197 - val_accuracy: 0.5586 Epoch 14/100 1148/1148 [==============================] - 24s 21ms/step - loss: 2.0513 - accuracy: 0.5343 - val_loss: 1.9661 - val_accuracy: 0.5666 Epoch 15/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.0050 - accuracy: 0.5421 - val_loss: 1.9241 - val_accuracy: 0.5749 Epoch 16/100 1148/1148 [==============================] - 27s 24ms/step - loss: 1.9625 - accuracy: 0.5490 - val_loss: 1.8881 - val_accuracy: 0.5844 Epoch 17/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.9223 - accuracy: 0.5584 - val_loss: 1.8492 - val_accuracy: 0.5894 Epoch 18/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.8984 - accuracy: 0.5622 - val_loss: 1.8181 - val_accuracy: 0.5958 Epoch 19/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.8565 - accuracy: 0.5688 - val_loss: 1.7843 - val_accuracy: 0.5989 Epoch 20/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.8344 - accuracy: 0.5706 - val_loss: 1.7636 - val_accuracy: 0.6065 Epoch 21/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.8046 - accuracy: 0.5778 - val_loss: 1.7598 - val_accuracy: 0.6078 Epoch 22/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.7887 - accuracy: 0.5812 - val_loss: 1.7331 - val_accuracy: 0.6098 Epoch 23/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.9282 - accuracy: 0.5573 - val_loss: 1.7927 - val_accuracy: 0.5976 Epoch 24/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.8093 - accuracy: 0.5754 - val_loss: 1.7203 - val_accuracy: 0.6103 Epoch 25/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.7649 - accuracy: 0.5841 - val_loss: 1.6917 - val_accuracy: 0.6185 Epoch 26/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.7289 - accuracy: 0.5918 - val_loss: 1.6734 - val_accuracy: 0.6205 Epoch 27/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.7112 - accuracy: 0.5936 - val_loss: 1.6605 - val_accuracy: 0.6232 Epoch 28/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.7344 - accuracy: 0.5924 - val_loss: 1.6490 - val_accuracy: 0.6204 Epoch 29/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.6877 - accuracy: 0.5960 - val_loss: 1.6245 - val_accuracy: 0.6323 Epoch 30/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.6682 - accuracy: 0.6032 - val_loss: 1.6244 - val_accuracy: 0.6263 Epoch 31/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.6592 - accuracy: 0.6050 - val_loss: 1.6076 - val_accuracy: 0.6311 Epoch 32/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.6466 - accuracy: 0.6075 - val_loss: 1.5948 - val_accuracy: 0.6351 Epoch 33/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.6383 - accuracy: 0.6099 - val_loss: 1.5931 - val_accuracy: 0.6313 Epoch 34/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.6244 - accuracy: 0.6136 - val_loss: 1.5884 - val_accuracy: 0.6397 Epoch 35/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.6131 - accuracy: 0.6150 - val_loss: 1.5739 - val_accuracy: 0.6411 Epoch 36/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.6510 - accuracy: 0.6073 - val_loss: 1.5565 - val_accuracy: 0.6438 Epoch 37/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5960 - accuracy: 0.6179 - val_loss: 1.5528 - val_accuracy: 0.6428 Epoch 38/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.5827 - accuracy: 0.6197 - val_loss: 1.5448 - val_accuracy: 0.6446 Epoch 39/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.5791 - accuracy: 0.6200 - val_loss: 1.5361 - val_accuracy: 0.6469 Epoch 40/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5719 - accuracy: 0.6203 - val_loss: 1.5373 - val_accuracy: 0.6531 Epoch 41/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5629 - accuracy: 0.6232 - val_loss: 1.5276 - val_accuracy: 0.6477 Epoch 42/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5570 - accuracy: 0.6252 - val_loss: 1.5133 - val_accuracy: 0.6520 Epoch 43/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5444 - accuracy: 0.6266 - val_loss: 1.5283 - val_accuracy: 0.6496 Epoch 44/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5423 - accuracy: 0.6307 - val_loss: 1.5146 - val_accuracy: 0.6505 Epoch 45/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5311 - accuracy: 0.6314 - val_loss: 1.5076 - val_accuracy: 0.6541 Epoch 46/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.5308 - accuracy: 0.6324 - val_loss: 1.5086 - val_accuracy: 0.6513 Epoch 47/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.5545 - accuracy: 0.6262 - val_loss: 1.4976 - val_accuracy: 0.6550 Epoch 48/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5055 - accuracy: 0.6368 - val_loss: 1.4899 - val_accuracy: 0.6563 Epoch 49/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.5120 - accuracy: 0.6327 - val_loss: 1.4808 - val_accuracy: 0.6617 Epoch 50/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5083 - accuracy: 0.6358 - val_loss: 1.4868 - val_accuracy: 0.6595 Epoch 51/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.5012 - accuracy: 0.6376 - val_loss: 1.4838 - val_accuracy: 0.6609 Epoch 52/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5035 - accuracy: 0.6375 - val_loss: 1.4800 - val_accuracy: 0.6612 Epoch 53/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.4883 - accuracy: 0.6413 - val_loss: 1.7156 - val_accuracy: 0.6196 Epoch 54/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.4931 - accuracy: 0.6392 - val_loss: 1.4851 - val_accuracy: 0.6567 Epoch 55/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4907 - accuracy: 0.6390 - val_loss: 1.4740 - val_accuracy: 0.6619 Epoch 56/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4754 - accuracy: 0.6422 - val_loss: 1.4722 - val_accuracy: 0.6643 Epoch 57/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.4676 - accuracy: 0.6417 - val_loss: 1.4695 - val_accuracy: 0.6611 Epoch 58/100 1148/1148 [==============================] - 28s 24ms/step - loss: 1.4732 - accuracy: 0.6449 - val_loss: 1.4675 - val_accuracy: 0.6635 Epoch 59/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4641 - accuracy: 0.6442 - val_loss: 1.4705 - val_accuracy: 0.6630 Epoch 60/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4722 - accuracy: 0.6436 - val_loss: 1.4686 - val_accuracy: 0.6626 Epoch 61/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.7771 - accuracy: 0.5929 - val_loss: 1.9123 - val_accuracy: 0.5775 Epoch 62/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.8656 - accuracy: 0.5569 - val_loss: 1.6042 - val_accuracy: 0.6295 Epoch 63/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.7869 - accuracy: 0.5767 - val_loss: 1.5681 - val_accuracy: 0.6362 Epoch 64/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.6156 - accuracy: 0.6062 - val_loss: 1.5194 - val_accuracy: 0.6509 Epoch 65/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5798 - accuracy: 0.6157 - val_loss: 1.5045 - val_accuracy: 0.6504 Epoch 66/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5535 - accuracy: 0.6197 - val_loss: 1.5272 - val_accuracy: 0.6508 Epoch 67/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5311 - accuracy: 0.6259 - val_loss: 1.4846 - val_accuracy: 0.6588 Epoch 68/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.7462 - accuracy: 0.5900 - val_loss: 1.5635 - val_accuracy: 0.6436 Epoch 69/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.5623 - accuracy: 0.6210 - val_loss: 1.4804 - val_accuracy: 0.6584 Epoch 70/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5237 - accuracy: 0.6275 - val_loss: 1.4673 - val_accuracy: 0.6617 Epoch 71/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.5090 - accuracy: 0.6314 - val_loss: 1.4605 - val_accuracy: 0.6628 Epoch 72/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4904 - accuracy: 0.6382 - val_loss: 1.4596 - val_accuracy: 0.6623 Epoch 73/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4801 - accuracy: 0.6389 - val_loss: 1.5789 - val_accuracy: 0.6478 Epoch 74/100 1148/1148 [==============================] - 27s 24ms/step - loss: 1.4701 - accuracy: 0.6419 - val_loss: 1.4397 - val_accuracy: 0.6676 Epoch 75/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4661 - accuracy: 0.6428 - val_loss: 1.4378 - val_accuracy: 0.6661 Epoch 76/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4601 - accuracy: 0.6454 - val_loss: 1.4384 - val_accuracy: 0.6669 Epoch 77/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4711 - accuracy: 0.6437 - val_loss: 1.4355 - val_accuracy: 0.6672 Epoch 78/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4451 - accuracy: 0.6451 - val_loss: 1.4276 - val_accuracy: 0.6705 Epoch 79/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4473 - accuracy: 0.6462 - val_loss: 1.4277 - val_accuracy: 0.6713 Epoch 80/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4397 - accuracy: 0.6474 - val_loss: 1.4342 - val_accuracy: 0.6722 Epoch 81/100 1148/1148 [==============================] - 27s 24ms/step - loss: 1.4378 - accuracy: 0.6492 - val_loss: 1.4178 - val_accuracy: 0.6659 Epoch 82/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4637 - accuracy: 0.6404 - val_loss: 1.4330 - val_accuracy: 0.6673 Epoch 83/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4238 - accuracy: 0.6521 - val_loss: 1.4144 - val_accuracy: 0.6711 Epoch 84/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4152 - accuracy: 0.6531 - val_loss: 1.4129 - val_accuracy: 0.6742 Epoch 85/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4185 - accuracy: 0.6545 - val_loss: 1.4261 - val_accuracy: 0.6719 Epoch 86/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4167 - accuracy: 0.6520 - val_loss: 1.4215 - val_accuracy: 0.6764 Epoch 87/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4123 - accuracy: 0.6552 - val_loss: 1.4134 - val_accuracy: 0.6727 Epoch 88/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4082 - accuracy: 0.6552 - val_loss: 1.4227 - val_accuracy: 0.6732 Epoch 89/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4104 - accuracy: 0.6572 - val_loss: 1.4073 - val_accuracy: 0.6775 Epoch 90/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.3993 - accuracy: 0.6572 - val_loss: 1.4054 - val_accuracy: 0.6751 Epoch 91/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.4070 - accuracy: 0.6578 - val_loss: 1.4091 - val_accuracy: 0.6755 Epoch 92/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.3971 - accuracy: 0.6609 - val_loss: 1.3971 - val_accuracy: 0.6806 Epoch 93/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.3991 - accuracy: 0.6598 - val_loss: 1.3981 - val_accuracy: 0.6733 Epoch 94/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.3872 - accuracy: 0.6619 - val_loss: 1.4000 - val_accuracy: 0.6760 Epoch 95/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.3934 - accuracy: 0.6573 - val_loss: 1.4022 - val_accuracy: 0.6779 Epoch 96/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.3914 - accuracy: 0.6616 - val_loss: 1.4082 - val_accuracy: 0.6780 Epoch 97/100 1148/1148 [==============================] - 28s 24ms/step - loss: 1.6727 - accuracy: 0.6161 - val_loss: 1.8350 - val_accuracy: 0.6014 Epoch 98/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.5965 - accuracy: 0.6178 - val_loss: 1.4349 - val_accuracy: 0.6711 Epoch 99/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.4624 - accuracy: 0.6454 - val_loss: 1.8151 - val_accuracy: 0.6084 Epoch 100/100 1148/1148 [==============================] - 27s 24ms/step - loss: 1.5589 - accuracy: 0.6221 - val_loss: 1.4305 - val_accuracy: 0.6692
simpleRNN.summary()
Model: "simpleRNN_v1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
simple_rnn (SimpleRNN) (None, 64) 4800
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 94,725
Trainable params: 94,725
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(simpleRNN_history.history)
Observations
simpleRNN.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 2s 4ms/step - loss: 1.4270 - accuracy: 0.6719
[1.4270331859588623, 0.6718698740005493]
Observations
tf.keras.backend.clear_session()
# Create the model
simpleRNNv2 = Sequential(
name='simpleRNN_v2',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
SimpleRNN(128, activation='tanh'),
Dropout(0.4),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
simpleRNNv2.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
simpleRNNv2_history = simpleRNNv2.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 26s 23ms/step - loss: 5.1854 - accuracy: 0.1266 - val_loss: 4.4167 - val_accuracy: 0.1887 Epoch 2/100 1148/1148 [==============================] - 26s 23ms/step - loss: 4.0057 - accuracy: 0.2419 - val_loss: 3.6041 - val_accuracy: 0.3003 Epoch 3/100 1148/1148 [==============================] - 26s 22ms/step - loss: 3.3374 - accuracy: 0.3295 - val_loss: 3.0882 - val_accuracy: 0.3775 Epoch 4/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.9006 - accuracy: 0.3909 - val_loss: 2.7336 - val_accuracy: 0.4318 Epoch 5/100 1148/1148 [==============================] - 27s 23ms/step - loss: 2.6046 - accuracy: 0.4399 - val_loss: 2.4872 - val_accuracy: 0.4776 Epoch 6/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.3724 - accuracy: 0.4805 - val_loss: 2.3047 - val_accuracy: 0.5096 Epoch 7/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.1957 - accuracy: 0.5117 - val_loss: 2.1323 - val_accuracy: 0.5422 Epoch 8/100 1148/1148 [==============================] - 26s 23ms/step - loss: 2.0613 - accuracy: 0.5295 - val_loss: 2.0262 - val_accuracy: 0.5595 Epoch 9/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.9411 - accuracy: 0.5561 - val_loss: 1.9233 - val_accuracy: 0.5809 Epoch 10/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.8377 - accuracy: 0.5737 - val_loss: 1.8386 - val_accuracy: 0.5971 Epoch 11/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.7705 - accuracy: 0.5889 - val_loss: 1.7749 - val_accuracy: 0.6099 Epoch 12/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.7035 - accuracy: 0.6008 - val_loss: 1.7846 - val_accuracy: 0.6026 Epoch 13/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.6439 - accuracy: 0.6142 - val_loss: 1.6781 - val_accuracy: 0.6289 Epoch 14/100 1148/1148 [==============================] - 31s 27ms/step - loss: 1.5879 - accuracy: 0.6236 - val_loss: 1.6231 - val_accuracy: 0.6362 Epoch 15/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.5453 - accuracy: 0.6320 - val_loss: 1.5839 - val_accuracy: 0.6456 Epoch 16/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.5868 - accuracy: 0.6266 - val_loss: 1.6351 - val_accuracy: 0.6341 Epoch 17/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.4943 - accuracy: 0.6426 - val_loss: 1.5357 - val_accuracy: 0.6560 Epoch 18/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4522 - accuracy: 0.6520 - val_loss: 1.5156 - val_accuracy: 0.6572 Epoch 19/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.4323 - accuracy: 0.6530 - val_loss: 1.5262 - val_accuracy: 0.6519 Epoch 20/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.4091 - accuracy: 0.6598 - val_loss: 1.4901 - val_accuracy: 0.6646 Epoch 21/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.4090 - accuracy: 0.6579 - val_loss: 1.4648 - val_accuracy: 0.6673 Epoch 22/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.3655 - accuracy: 0.6689 - val_loss: 1.4711 - val_accuracy: 0.6688 Epoch 23/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.3573 - accuracy: 0.6731 - val_loss: 1.4667 - val_accuracy: 0.6714 Epoch 24/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.3429 - accuracy: 0.6724 - val_loss: 1.4286 - val_accuracy: 0.6757 Epoch 25/100 1148/1148 [==============================] - 28s 24ms/step - loss: 1.3231 - accuracy: 0.6773 - val_loss: 1.4190 - val_accuracy: 0.6762 Epoch 26/100 1148/1148 [==============================] - 27s 24ms/step - loss: 1.3154 - accuracy: 0.6803 - val_loss: 1.4244 - val_accuracy: 0.6745 Epoch 27/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.2967 - accuracy: 0.6824 - val_loss: 1.4175 - val_accuracy: 0.6754 Epoch 28/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.3323 - accuracy: 0.6776 - val_loss: 1.4058 - val_accuracy: 0.6789 Epoch 29/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2758 - accuracy: 0.6911 - val_loss: 1.3934 - val_accuracy: 0.6832 Epoch 30/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2718 - accuracy: 0.6870 - val_loss: 1.3760 - val_accuracy: 0.6916 Epoch 31/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2665 - accuracy: 0.6913 - val_loss: 1.3953 - val_accuracy: 0.6814 Epoch 32/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2714 - accuracy: 0.6880 - val_loss: 1.3838 - val_accuracy: 0.6867 Epoch 33/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2381 - accuracy: 0.6957 - val_loss: 1.3821 - val_accuracy: 0.6799 Epoch 34/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2386 - accuracy: 0.6974 - val_loss: 1.3690 - val_accuracy: 0.6863 Epoch 35/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2251 - accuracy: 0.7009 - val_loss: 1.3654 - val_accuracy: 0.6867 Epoch 36/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2337 - accuracy: 0.6955 - val_loss: 1.3761 - val_accuracy: 0.6896 Epoch 37/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2100 - accuracy: 0.7017 - val_loss: 1.3526 - val_accuracy: 0.6878 Epoch 38/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.2232 - accuracy: 0.6992 - val_loss: 1.5633 - val_accuracy: 0.6588 Epoch 39/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.2678 - accuracy: 0.6897 - val_loss: 1.3526 - val_accuracy: 0.6894 Epoch 40/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.2048 - accuracy: 0.7031 - val_loss: 1.3480 - val_accuracy: 0.6921 Epoch 41/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2174 - accuracy: 0.7007 - val_loss: 1.3519 - val_accuracy: 0.6906 Epoch 42/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1857 - accuracy: 0.7052 - val_loss: 1.3388 - val_accuracy: 0.6930 Epoch 43/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.1874 - accuracy: 0.7072 - val_loss: 1.3379 - val_accuracy: 0.6946 Epoch 44/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1840 - accuracy: 0.7096 - val_loss: 1.3289 - val_accuracy: 0.6954 Epoch 45/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.1813 - accuracy: 0.7072 - val_loss: 1.3403 - val_accuracy: 0.6926 Epoch 46/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1786 - accuracy: 0.7097 - val_loss: 1.3436 - val_accuracy: 0.6959 Epoch 47/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1715 - accuracy: 0.7114 - val_loss: 1.3381 - val_accuracy: 0.6952 Epoch 48/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1789 - accuracy: 0.7105 - val_loss: 2.0287 - val_accuracy: 0.5812 Epoch 49/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.3046 - accuracy: 0.6840 - val_loss: 1.3291 - val_accuracy: 0.6978 Epoch 50/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.1451 - accuracy: 0.7153 - val_loss: 1.3319 - val_accuracy: 0.6993 Epoch 51/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1472 - accuracy: 0.7146 - val_loss: 1.3229 - val_accuracy: 0.6984 Epoch 52/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.1539 - accuracy: 0.7142 - val_loss: 1.3276 - val_accuracy: 0.6986 Epoch 53/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1473 - accuracy: 0.7170 - val_loss: 1.3356 - val_accuracy: 0.6928 Epoch 54/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2141 - accuracy: 0.7038 - val_loss: 1.3189 - val_accuracy: 0.7021 Epoch 55/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1384 - accuracy: 0.7155 - val_loss: 1.3265 - val_accuracy: 0.7004 Epoch 56/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.1394 - accuracy: 0.7172 - val_loss: 1.3364 - val_accuracy: 0.7016 Epoch 57/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.3092 - accuracy: 0.6885 - val_loss: 1.4532 - val_accuracy: 0.6835 Epoch 58/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.1976 - accuracy: 0.7077 - val_loss: 1.3130 - val_accuracy: 0.7019 Epoch 59/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1348 - accuracy: 0.7185 - val_loss: 1.3193 - val_accuracy: 0.7028 Epoch 60/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.1344 - accuracy: 0.7177 - val_loss: 1.3135 - val_accuracy: 0.7022 Epoch 61/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1451 - accuracy: 0.7184 - val_loss: 1.3059 - val_accuracy: 0.7041 Epoch 62/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1274 - accuracy: 0.7215 - val_loss: 1.3268 - val_accuracy: 0.7019 Epoch 63/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2561 - accuracy: 0.6962 - val_loss: 1.3263 - val_accuracy: 0.6999 Epoch 64/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1335 - accuracy: 0.7205 - val_loss: 1.3035 - val_accuracy: 0.7063 Epoch 65/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.1236 - accuracy: 0.7226 - val_loss: 1.3161 - val_accuracy: 0.7008 Epoch 66/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1541 - accuracy: 0.7152 - val_loss: 1.5255 - val_accuracy: 0.6649 Epoch 67/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1653 - accuracy: 0.7139 - val_loss: 1.3169 - val_accuracy: 0.7040 Epoch 68/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.1224 - accuracy: 0.7228 - val_loss: 1.3142 - val_accuracy: 0.7056 Epoch 69/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.2207 - accuracy: 0.7051 - val_loss: 1.3221 - val_accuracy: 0.7023 Epoch 70/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1423 - accuracy: 0.7182 - val_loss: 1.3462 - val_accuracy: 0.6994 Epoch 71/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1228 - accuracy: 0.7219 - val_loss: 1.3063 - val_accuracy: 0.7068 Epoch 72/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1622 - accuracy: 0.7173 - val_loss: 1.4063 - val_accuracy: 0.6854 Epoch 73/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1869 - accuracy: 0.7095 - val_loss: 1.3048 - val_accuracy: 0.7050 Epoch 74/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.1092 - accuracy: 0.7252 - val_loss: 1.2985 - val_accuracy: 0.7067 Epoch 75/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1118 - accuracy: 0.7252 - val_loss: 1.3109 - val_accuracy: 0.7046 Epoch 76/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1060 - accuracy: 0.7264 - val_loss: 1.2947 - val_accuracy: 0.7100 Epoch 77/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1028 - accuracy: 0.7260 - val_loss: 1.2986 - val_accuracy: 0.7100 Epoch 78/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1080 - accuracy: 0.7273 - val_loss: 1.3019 - val_accuracy: 0.7073 Epoch 79/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1037 - accuracy: 0.7266 - val_loss: 1.2962 - val_accuracy: 0.7066 Epoch 80/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1030 - accuracy: 0.7269 - val_loss: 1.3042 - val_accuracy: 0.7042 Epoch 81/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.3897 - accuracy: 0.6760 - val_loss: 1.3891 - val_accuracy: 0.6898 Epoch 82/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.2160 - accuracy: 0.7043 - val_loss: 1.3039 - val_accuracy: 0.7069 Epoch 83/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0974 - accuracy: 0.7260 - val_loss: 1.2972 - val_accuracy: 0.7055 Epoch 84/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.0924 - accuracy: 0.7291 - val_loss: 1.3013 - val_accuracy: 0.7018 Epoch 85/100 1148/1148 [==============================] - 26s 22ms/step - loss: 1.1004 - accuracy: 0.7265 - val_loss: 1.2984 - val_accuracy: 0.7049 Epoch 86/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0944 - accuracy: 0.7287 - val_loss: 1.2913 - val_accuracy: 0.7064 Epoch 87/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.1962 - accuracy: 0.7107 - val_loss: 1.2960 - val_accuracy: 0.7092 Epoch 88/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0891 - accuracy: 0.7313 - val_loss: 1.3005 - val_accuracy: 0.7064 Epoch 89/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0880 - accuracy: 0.7287 - val_loss: 1.2977 - val_accuracy: 0.7101 Epoch 90/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0904 - accuracy: 0.7277 - val_loss: 1.3163 - val_accuracy: 0.7060 Epoch 91/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0942 - accuracy: 0.7312 - val_loss: 1.2822 - val_accuracy: 0.7069 Epoch 92/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.1070 - accuracy: 0.7282 - val_loss: 1.2837 - val_accuracy: 0.7104 Epoch 93/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.0809 - accuracy: 0.7299 - val_loss: 1.2893 - val_accuracy: 0.7059 Epoch 94/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0924 - accuracy: 0.7280 - val_loss: 1.2983 - val_accuracy: 0.7058 Epoch 95/100 1148/1148 [==============================] - 24s 21ms/step - loss: 1.0786 - accuracy: 0.7310 - val_loss: 1.2917 - val_accuracy: 0.7068 Epoch 96/100 1148/1148 [==============================] - 25s 22ms/step - loss: 1.0809 - accuracy: 0.7306 - val_loss: 1.3031 - val_accuracy: 0.7068 Epoch 97/100 1148/1148 [==============================] - 25s 21ms/step - loss: 1.0848 - accuracy: 0.7328 - val_loss: 1.2949 - val_accuracy: 0.7038 Epoch 98/100 1148/1148 [==============================] - 22s 19ms/step - loss: 1.0745 - accuracy: 0.7320 - val_loss: 1.2814 - val_accuracy: 0.7090 Epoch 99/100 1148/1148 [==============================] - 26s 23ms/step - loss: 1.0724 - accuracy: 0.7334 - val_loss: 1.3114 - val_accuracy: 0.7044 Epoch 100/100 1148/1148 [==============================] - 27s 23ms/step - loss: 1.0747 - accuracy: 0.7342 - val_loss: 1.2805 - val_accuracy: 0.7111
simpleRNNv2.summary()
Model: "simpleRNN_v2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
simple_rnn (SimpleRNN) (None, 128) 17792
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 184,453
Trainable params: 184,453
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(simpleRNNv2_history.history)
Observations
simpleRNNv2.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 4ms/step - loss: 1.2719 - accuracy: 0.7100
[1.2719430923461914, 0.7099542617797852]
Observations
tf.keras.backend.clear_session()
# Create the model
simpleRNNv3 = Sequential(
name='simpleRNN_v3',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
SimpleRNN(128, activation='tanh', return_sequences=True),
SimpleRNN(64, activation='tanh'),
Dropout(0.4),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
simpleRNNv3.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
simpleRNNv3_history = simpleRNNv3.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 50s 43ms/step - loss: 5.2110 - accuracy: 0.1101 - val_loss: 4.6106 - val_accuracy: 0.1612 Epoch 2/100 1148/1148 [==============================] - 43s 38ms/step - loss: 4.2475 - accuracy: 0.2097 - val_loss: 3.8483 - val_accuracy: 0.2694 Epoch 3/100 1148/1148 [==============================] - 44s 38ms/step - loss: 3.6423 - accuracy: 0.2870 - val_loss: 3.3859 - val_accuracy: 0.3363 Epoch 4/100 1148/1148 [==============================] - 44s 38ms/step - loss: 3.2731 - accuracy: 0.3379 - val_loss: 3.0856 - val_accuracy: 0.3793 Epoch 5/100 1148/1148 [==============================] - 43s 37ms/step - loss: 3.0079 - accuracy: 0.3786 - val_loss: 2.8470 - val_accuracy: 0.4155 Epoch 6/100 1148/1148 [==============================] - 45s 39ms/step - loss: 2.8032 - accuracy: 0.4096 - val_loss: 2.7070 - val_accuracy: 0.4394 Epoch 7/100 1148/1148 [==============================] - 46s 40ms/step - loss: 2.6457 - accuracy: 0.4348 - val_loss: 2.5229 - val_accuracy: 0.4720 Epoch 8/100 1148/1148 [==============================] - 48s 42ms/step - loss: 2.5126 - accuracy: 0.4583 - val_loss: 2.3986 - val_accuracy: 0.4965 Epoch 9/100 1148/1148 [==============================] - 44s 38ms/step - loss: 2.3943 - accuracy: 0.4800 - val_loss: 2.2775 - val_accuracy: 0.5126 Epoch 10/100 1148/1148 [==============================] - 44s 38ms/step - loss: 2.2906 - accuracy: 0.4976 - val_loss: 2.1909 - val_accuracy: 0.5385 Epoch 11/100 1148/1148 [==============================] - 43s 38ms/step - loss: 2.2148 - accuracy: 0.5125 - val_loss: 2.1307 - val_accuracy: 0.5509 Epoch 12/100 1148/1148 [==============================] - 43s 38ms/step - loss: 2.1463 - accuracy: 0.5255 - val_loss: 2.0532 - val_accuracy: 0.5663 Epoch 13/100 1148/1148 [==============================] - 44s 38ms/step - loss: 2.0743 - accuracy: 0.5402 - val_loss: 1.9919 - val_accuracy: 0.5757 Epoch 14/100 1148/1148 [==============================] - 44s 38ms/step - loss: 2.0216 - accuracy: 0.5488 - val_loss: 1.9339 - val_accuracy: 0.5861 Epoch 15/100 1148/1148 [==============================] - 43s 38ms/step - loss: 1.9719 - accuracy: 0.5581 - val_loss: 1.8960 - val_accuracy: 0.5928 Epoch 16/100 1148/1148 [==============================] - 48s 42ms/step - loss: 1.9441 - accuracy: 0.5636 - val_loss: 1.8760 - val_accuracy: 0.5941 Epoch 17/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.8902 - accuracy: 0.5727 - val_loss: 1.8203 - val_accuracy: 0.6073 Epoch 18/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.8636 - accuracy: 0.5787 - val_loss: 1.7977 - val_accuracy: 0.6073 Epoch 19/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.8297 - accuracy: 0.5859 - val_loss: 1.7713 - val_accuracy: 0.6188 Epoch 20/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.8095 - accuracy: 0.5894 - val_loss: 1.7317 - val_accuracy: 0.6252 Epoch 21/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.7912 - accuracy: 0.5953 - val_loss: 1.7460 - val_accuracy: 0.6268 Epoch 22/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.7521 - accuracy: 0.6029 - val_loss: 1.6929 - val_accuracy: 0.6339 Epoch 23/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.7337 - accuracy: 0.6056 - val_loss: 1.6811 - val_accuracy: 0.6350 Epoch 24/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.7250 - accuracy: 0.6069 - val_loss: 1.6654 - val_accuracy: 0.6334 Epoch 25/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.6853 - accuracy: 0.6164 - val_loss: 1.6344 - val_accuracy: 0.6460 Epoch 26/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.6717 - accuracy: 0.6167 - val_loss: 1.6599 - val_accuracy: 0.6374 Epoch 27/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.6603 - accuracy: 0.6226 - val_loss: 1.6183 - val_accuracy: 0.6511 Epoch 28/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.6476 - accuracy: 0.6229 - val_loss: 1.6249 - val_accuracy: 0.6498 Epoch 29/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.6258 - accuracy: 0.6272 - val_loss: 1.7436 - val_accuracy: 0.6260 Epoch 30/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.6269 - accuracy: 0.6251 - val_loss: 1.5921 - val_accuracy: 0.6556 Epoch 31/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5881 - accuracy: 0.6354 - val_loss: 1.5718 - val_accuracy: 0.6544 Epoch 32/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.5763 - accuracy: 0.6365 - val_loss: 1.5743 - val_accuracy: 0.6565 Epoch 33/100 1148/1148 [==============================] - 46s 41ms/step - loss: 1.5750 - accuracy: 0.6346 - val_loss: 1.5775 - val_accuracy: 0.6531 Epoch 34/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5633 - accuracy: 0.6410 - val_loss: 1.5592 - val_accuracy: 0.6588 Epoch 35/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5468 - accuracy: 0.6443 - val_loss: 1.5410 - val_accuracy: 0.6633 Epoch 36/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.5514 - accuracy: 0.6475 - val_loss: 1.5519 - val_accuracy: 0.6668 Epoch 37/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5376 - accuracy: 0.6491 - val_loss: 1.5397 - val_accuracy: 0.6626 Epoch 38/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.5468 - accuracy: 0.6467 - val_loss: 1.5263 - val_accuracy: 0.6635 Epoch 39/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.5148 - accuracy: 0.6498 - val_loss: 1.5081 - val_accuracy: 0.6645 Epoch 40/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5188 - accuracy: 0.6497 - val_loss: 1.4957 - val_accuracy: 0.6726 Epoch 41/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5001 - accuracy: 0.6552 - val_loss: 1.5141 - val_accuracy: 0.6715 Epoch 42/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.4953 - accuracy: 0.6566 - val_loss: 1.5069 - val_accuracy: 0.6723 Epoch 43/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.6110 - accuracy: 0.6339 - val_loss: 1.6969 - val_accuracy: 0.6285 Epoch 44/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5151 - accuracy: 0.6490 - val_loss: 1.4732 - val_accuracy: 0.6787 Epoch 45/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4649 - accuracy: 0.6655 - val_loss: 1.5003 - val_accuracy: 0.6711 Epoch 46/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.4629 - accuracy: 0.6609 - val_loss: 1.5053 - val_accuracy: 0.6753 Epoch 47/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4529 - accuracy: 0.6660 - val_loss: 1.4856 - val_accuracy: 0.6736 Epoch 48/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.4469 - accuracy: 0.6674 - val_loss: 1.4832 - val_accuracy: 0.6756 Epoch 49/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4682 - accuracy: 0.6604 - val_loss: 1.4604 - val_accuracy: 0.6830 Epoch 50/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5202 - accuracy: 0.6527 - val_loss: 1.5677 - val_accuracy: 0.6563 Epoch 51/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4571 - accuracy: 0.6625 - val_loss: 1.4637 - val_accuracy: 0.6782 Epoch 52/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.4480 - accuracy: 0.6659 - val_loss: 1.6914 - val_accuracy: 0.6424 Epoch 53/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.5495 - accuracy: 0.6457 - val_loss: 1.4599 - val_accuracy: 0.6813 Epoch 54/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.4125 - accuracy: 0.6747 - val_loss: 1.5618 - val_accuracy: 0.6636 Epoch 55/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.4370 - accuracy: 0.6684 - val_loss: 1.4564 - val_accuracy: 0.6810 Epoch 56/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.4065 - accuracy: 0.6767 - val_loss: 1.4517 - val_accuracy: 0.6796 Epoch 57/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.4726 - accuracy: 0.6634 - val_loss: 1.4719 - val_accuracy: 0.6795 Epoch 58/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.4068 - accuracy: 0.6768 - val_loss: 1.4457 - val_accuracy: 0.6839 Epoch 59/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4073 - accuracy: 0.6760 - val_loss: 1.4381 - val_accuracy: 0.6841 Epoch 60/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3937 - accuracy: 0.6775 - val_loss: 1.4330 - val_accuracy: 0.6887 Epoch 61/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3962 - accuracy: 0.6788 - val_loss: 1.4462 - val_accuracy: 0.6876 Epoch 62/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.4240 - accuracy: 0.6750 - val_loss: 1.4405 - val_accuracy: 0.6901 Epoch 63/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3767 - accuracy: 0.6832 - val_loss: 1.4761 - val_accuracy: 0.6822 Epoch 64/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3792 - accuracy: 0.6812 - val_loss: 1.4284 - val_accuracy: 0.6858 Epoch 65/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3864 - accuracy: 0.6796 - val_loss: 1.4424 - val_accuracy: 0.6855 Epoch 66/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3918 - accuracy: 0.6785 - val_loss: 1.5418 - val_accuracy: 0.6733 Epoch 67/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.4371 - accuracy: 0.6732 - val_loss: 1.4261 - val_accuracy: 0.6902 Epoch 68/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3629 - accuracy: 0.6844 - val_loss: 1.4097 - val_accuracy: 0.6904 Epoch 69/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3516 - accuracy: 0.6857 - val_loss: 1.4443 - val_accuracy: 0.6843 Epoch 70/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3685 - accuracy: 0.6850 - val_loss: 1.4256 - val_accuracy: 0.6880 Epoch 71/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3441 - accuracy: 0.6907 - val_loss: 1.4084 - val_accuracy: 0.6908 Epoch 72/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3631 - accuracy: 0.6869 - val_loss: 1.4794 - val_accuracy: 0.6751 Epoch 73/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3582 - accuracy: 0.6861 - val_loss: 1.4170 - val_accuracy: 0.6887 Epoch 74/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3485 - accuracy: 0.6863 - val_loss: 1.4351 - val_accuracy: 0.6893 Epoch 75/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3437 - accuracy: 0.6915 - val_loss: 1.4102 - val_accuracy: 0.6924 Epoch 76/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3444 - accuracy: 0.6910 - val_loss: 1.4120 - val_accuracy: 0.6928 Epoch 77/100 1148/1148 [==============================] - 46s 40ms/step - loss: 1.3243 - accuracy: 0.6944 - val_loss: 1.4171 - val_accuracy: 0.6902 Epoch 78/100 1148/1148 [==============================] - 44s 39ms/step - loss: 1.5582 - accuracy: 0.6478 - val_loss: 1.4534 - val_accuracy: 0.6819 Epoch 79/100 1148/1148 [==============================] - 50s 44ms/step - loss: 1.3590 - accuracy: 0.6876 - val_loss: 1.4001 - val_accuracy: 0.6903 Epoch 80/100 1148/1148 [==============================] - 48s 42ms/step - loss: 1.3163 - accuracy: 0.6961 - val_loss: 1.4147 - val_accuracy: 0.6917 Epoch 81/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3320 - accuracy: 0.6916 - val_loss: 1.4255 - val_accuracy: 0.6893 Epoch 82/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.3341 - accuracy: 0.6932 - val_loss: 1.4042 - val_accuracy: 0.6906 Epoch 83/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3192 - accuracy: 0.6939 - val_loss: 1.4009 - val_accuracy: 0.6888 Epoch 84/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3099 - accuracy: 0.6971 - val_loss: 1.3947 - val_accuracy: 0.6897 Epoch 85/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.5130 - accuracy: 0.6571 - val_loss: 1.4151 - val_accuracy: 0.6873 Epoch 86/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.3143 - accuracy: 0.6968 - val_loss: 1.4013 - val_accuracy: 0.6934 Epoch 87/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.3141 - accuracy: 0.6976 - val_loss: 1.3952 - val_accuracy: 0.6939 Epoch 88/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3022 - accuracy: 0.6972 - val_loss: 1.4006 - val_accuracy: 0.6907 Epoch 89/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.4060 - accuracy: 0.6797 - val_loss: 1.5046 - val_accuracy: 0.6802 Epoch 90/100 1148/1148 [==============================] - 44s 38ms/step - loss: 1.3065 - accuracy: 0.6999 - val_loss: 1.5429 - val_accuracy: 0.6731 Epoch 91/100 1148/1148 [==============================] - 44s 39ms/step - loss: 1.6908 - accuracy: 0.6203 - val_loss: 1.4499 - val_accuracy: 0.6840 Epoch 92/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.3294 - accuracy: 0.6903 - val_loss: 1.3916 - val_accuracy: 0.6905 Epoch 93/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.2913 - accuracy: 0.7020 - val_loss: 1.4021 - val_accuracy: 0.6936 Epoch 94/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.2964 - accuracy: 0.7002 - val_loss: 1.4027 - val_accuracy: 0.6951 Epoch 95/100 1148/1148 [==============================] - 50s 44ms/step - loss: 1.5178 - accuracy: 0.6578 - val_loss: 1.6553 - val_accuracy: 0.6400 Epoch 96/100 1148/1148 [==============================] - 47s 41ms/step - loss: 1.4293 - accuracy: 0.6717 - val_loss: 1.4043 - val_accuracy: 0.6947 Epoch 97/100 1148/1148 [==============================] - 49s 43ms/step - loss: 1.3073 - accuracy: 0.6971 - val_loss: 1.3889 - val_accuracy: 0.6952 Epoch 98/100 1148/1148 [==============================] - 45s 39ms/step - loss: 1.2947 - accuracy: 0.7006 - val_loss: 1.4065 - val_accuracy: 0.6953 Epoch 99/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.3211 - accuracy: 0.6943 - val_loss: 1.3989 - val_accuracy: 0.6957 Epoch 100/100 1148/1148 [==============================] - 45s 40ms/step - loss: 1.2889 - accuracy: 0.7013 - val_loss: 1.3993 - val_accuracy: 0.6952
simpleRNNv3.summary()
Model: "simpleRNN_v3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
simple_rnn (SimpleRNN) (None, 34, 128) 17792
simple_rnn_1 (SimpleRNN) (None, 64) 12352
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 120,069
Trainable params: 120,069
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(simpleRNNv3_history.history)
Observations
simpleRNNv3.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 2s 6ms/step - loss: 1.3865 - accuracy: 0.6946
[1.386472463607788, 0.6945897340774536]
Observations

Image Source: Prashant Banerjee, 2019

Image Source: Siddharth, M., 2021
tf.keras.backend.clear_session()
# Create the model
LSTM_V1 = Sequential(
name='lstm_v1',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
LSTM(256, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
LSTM_V1.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
LSTM_V1_history = LSTM_V1.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 8s 6ms/step - loss: 5.0829 - accuracy: 0.1221 - val_loss: 4.5392 - val_accuracy: 0.1528 Epoch 2/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.1428 - accuracy: 0.2095 - val_loss: 3.7846 - val_accuracy: 0.2632 Epoch 3/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4359 - accuracy: 0.3003 - val_loss: 3.1999 - val_accuracy: 0.3425 Epoch 4/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9136 - accuracy: 0.3804 - val_loss: 2.7889 - val_accuracy: 0.4202 Epoch 5/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5276 - accuracy: 0.4465 - val_loss: 2.4847 - val_accuracy: 0.4736 Epoch 6/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2489 - accuracy: 0.4947 - val_loss: 2.2676 - val_accuracy: 0.5072 Epoch 7/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0388 - accuracy: 0.5303 - val_loss: 2.1052 - val_accuracy: 0.5428 Epoch 8/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8768 - accuracy: 0.5606 - val_loss: 1.9694 - val_accuracy: 0.5651 Epoch 9/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7383 - accuracy: 0.5865 - val_loss: 1.8805 - val_accuracy: 0.5823 Epoch 10/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6278 - accuracy: 0.6094 - val_loss: 1.7882 - val_accuracy: 0.6014 Epoch 11/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5361 - accuracy: 0.6268 - val_loss: 1.7254 - val_accuracy: 0.6149 Epoch 12/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4554 - accuracy: 0.6440 - val_loss: 1.6713 - val_accuracy: 0.6231 Epoch 13/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3885 - accuracy: 0.6555 - val_loss: 1.6270 - val_accuracy: 0.6311 Epoch 14/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3321 - accuracy: 0.6657 - val_loss: 1.5983 - val_accuracy: 0.6372 Epoch 15/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2832 - accuracy: 0.6783 - val_loss: 1.5682 - val_accuracy: 0.6469 Epoch 16/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2343 - accuracy: 0.6880 - val_loss: 1.5327 - val_accuracy: 0.6543 Epoch 17/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2026 - accuracy: 0.6970 - val_loss: 1.5111 - val_accuracy: 0.6573 Epoch 18/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1669 - accuracy: 0.7049 - val_loss: 1.5034 - val_accuracy: 0.6572 Epoch 19/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1333 - accuracy: 0.7110 - val_loss: 1.4878 - val_accuracy: 0.6619 Epoch 20/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1009 - accuracy: 0.7170 - val_loss: 1.4639 - val_accuracy: 0.6666 Epoch 21/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0761 - accuracy: 0.7220 - val_loss: 1.4588 - val_accuracy: 0.6671 Epoch 22/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0545 - accuracy: 0.7267 - val_loss: 1.4548 - val_accuracy: 0.6693 Epoch 23/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0386 - accuracy: 0.7309 - val_loss: 1.4530 - val_accuracy: 0.6728 Epoch 24/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0149 - accuracy: 0.7335 - val_loss: 1.4385 - val_accuracy: 0.6706 Epoch 25/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0008 - accuracy: 0.7389 - val_loss: 1.4334 - val_accuracy: 0.6720 Epoch 26/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9836 - accuracy: 0.7417 - val_loss: 1.4228 - val_accuracy: 0.6778 Epoch 27/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9692 - accuracy: 0.7434 - val_loss: 1.4205 - val_accuracy: 0.6773 Epoch 28/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9529 - accuracy: 0.7468 - val_loss: 1.4313 - val_accuracy: 0.6753 Epoch 29/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9439 - accuracy: 0.7498 - val_loss: 1.4238 - val_accuracy: 0.6812 Epoch 30/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9287 - accuracy: 0.7517 - val_loss: 1.4345 - val_accuracy: 0.6818 Epoch 31/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9175 - accuracy: 0.7559 - val_loss: 1.4234 - val_accuracy: 0.6849 Epoch 32/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9147 - accuracy: 0.7554 - val_loss: 1.4253 - val_accuracy: 0.6810 Epoch 33/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9026 - accuracy: 0.7590 - val_loss: 1.4202 - val_accuracy: 0.6863 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8946 - accuracy: 0.7611 - val_loss: 1.4317 - val_accuracy: 0.6845 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8870 - accuracy: 0.7619 - val_loss: 1.4182 - val_accuracy: 0.6855 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8771 - accuracy: 0.7626 - val_loss: 1.4348 - val_accuracy: 0.6808 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8695 - accuracy: 0.7642 - val_loss: 1.4310 - val_accuracy: 0.6870 Epoch 38/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8643 - accuracy: 0.7674 - val_loss: 1.4319 - val_accuracy: 0.6861 Epoch 39/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8539 - accuracy: 0.7691 - val_loss: 1.4335 - val_accuracy: 0.6870 Epoch 40/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8504 - accuracy: 0.7700 - val_loss: 1.4399 - val_accuracy: 0.6874 Epoch 41/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8470 - accuracy: 0.7690 - val_loss: 1.4284 - val_accuracy: 0.6885 Epoch 42/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8424 - accuracy: 0.7713 - val_loss: 1.4298 - val_accuracy: 0.6867 Epoch 43/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8383 - accuracy: 0.7710 - val_loss: 1.4331 - val_accuracy: 0.6872 Epoch 44/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8304 - accuracy: 0.7718 - val_loss: 1.4296 - val_accuracy: 0.6904 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8315 - accuracy: 0.7719 - val_loss: 1.4352 - val_accuracy: 0.6914 Epoch 46/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8196 - accuracy: 0.7754 - val_loss: 1.4448 - val_accuracy: 0.6902 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8183 - accuracy: 0.7757 - val_loss: 1.4496 - val_accuracy: 0.6898 Epoch 48/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8143 - accuracy: 0.7764 - val_loss: 1.4520 - val_accuracy: 0.6904 Epoch 49/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8142 - accuracy: 0.7758 - val_loss: 1.4578 - val_accuracy: 0.6892 Epoch 50/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8076 - accuracy: 0.7767 - val_loss: 1.4577 - val_accuracy: 0.6933 Epoch 51/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8030 - accuracy: 0.7793 - val_loss: 1.4692 - val_accuracy: 0.6903 Epoch 52/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8012 - accuracy: 0.7797 - val_loss: 1.4761 - val_accuracy: 0.6912 Epoch 53/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7968 - accuracy: 0.7804 - val_loss: 1.4614 - val_accuracy: 0.6890 Epoch 54/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7977 - accuracy: 0.7796 - val_loss: 1.4720 - val_accuracy: 0.6913 Epoch 55/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7929 - accuracy: 0.7793 - val_loss: 1.4805 - val_accuracy: 0.6932 Epoch 56/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7933 - accuracy: 0.7772 - val_loss: 1.4671 - val_accuracy: 0.6906 Epoch 57/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7828 - accuracy: 0.7812 - val_loss: 1.4758 - val_accuracy: 0.6921 Epoch 58/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7866 - accuracy: 0.7799 - val_loss: 1.4668 - val_accuracy: 0.6912 Epoch 59/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7874 - accuracy: 0.7792 - val_loss: 1.4805 - val_accuracy: 0.6941 Epoch 60/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7759 - accuracy: 0.7830 - val_loss: 1.4874 - val_accuracy: 0.6899 Epoch 61/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7767 - accuracy: 0.7825 - val_loss: 1.4834 - val_accuracy: 0.6939 Epoch 62/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7715 - accuracy: 0.7829 - val_loss: 1.4916 - val_accuracy: 0.6932 Epoch 63/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7742 - accuracy: 0.7819 - val_loss: 1.5020 - val_accuracy: 0.6902 Epoch 64/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7663 - accuracy: 0.7838 - val_loss: 1.4867 - val_accuracy: 0.6930 Epoch 65/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7678 - accuracy: 0.7843 - val_loss: 1.4945 - val_accuracy: 0.6943 Epoch 66/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7673 - accuracy: 0.7840 - val_loss: 1.4810 - val_accuracy: 0.6923 Epoch 67/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7651 - accuracy: 0.7829 - val_loss: 1.5019 - val_accuracy: 0.6899 Epoch 68/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7623 - accuracy: 0.7857 - val_loss: 1.5050 - val_accuracy: 0.6932 Epoch 69/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7589 - accuracy: 0.7865 - val_loss: 1.5006 - val_accuracy: 0.6920 Epoch 70/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7568 - accuracy: 0.7858 - val_loss: 1.5018 - val_accuracy: 0.6932 Epoch 71/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7626 - accuracy: 0.7855 - val_loss: 1.5020 - val_accuracy: 0.6939 Epoch 72/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7561 - accuracy: 0.7861 - val_loss: 1.4945 - val_accuracy: 0.6927 Epoch 73/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7532 - accuracy: 0.7872 - val_loss: 1.5086 - val_accuracy: 0.6930 Epoch 74/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7522 - accuracy: 0.7869 - val_loss: 1.5025 - val_accuracy: 0.6940 Epoch 75/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7557 - accuracy: 0.7877 - val_loss: 1.4905 - val_accuracy: 0.6935 Epoch 76/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7475 - accuracy: 0.7873 - val_loss: 1.5018 - val_accuracy: 0.6889 Epoch 77/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7486 - accuracy: 0.7868 - val_loss: 1.5131 - val_accuracy: 0.6931 Epoch 78/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7492 - accuracy: 0.7890 - val_loss: 1.5208 - val_accuracy: 0.6927 Epoch 79/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7438 - accuracy: 0.7893 - val_loss: 1.5151 - val_accuracy: 0.6929 Epoch 80/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7412 - accuracy: 0.7869 - val_loss: 1.5154 - val_accuracy: 0.6929 Epoch 81/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7447 - accuracy: 0.7868 - val_loss: 1.5201 - val_accuracy: 0.6895 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7455 - accuracy: 0.7876 - val_loss: 1.5299 - val_accuracy: 0.6952 Epoch 83/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.7378 - accuracy: 0.7891 - val_loss: 1.5345 - val_accuracy: 0.6904 Epoch 84/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7373 - accuracy: 0.7902 - val_loss: 1.5276 - val_accuracy: 0.6957 Epoch 85/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7367 - accuracy: 0.7897 - val_loss: 1.5277 - val_accuracy: 0.6914 Epoch 86/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7361 - accuracy: 0.7898 - val_loss: 1.5233 - val_accuracy: 0.6976 Epoch 87/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7381 - accuracy: 0.7882 - val_loss: 1.5382 - val_accuracy: 0.6933 Epoch 88/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7397 - accuracy: 0.7889 - val_loss: 1.5362 - val_accuracy: 0.6932 Epoch 89/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7402 - accuracy: 0.7879 - val_loss: 1.5323 - val_accuracy: 0.6931 Epoch 90/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7316 - accuracy: 0.7899 - val_loss: 1.5314 - val_accuracy: 0.6970 Epoch 91/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7335 - accuracy: 0.7891 - val_loss: 1.5399 - val_accuracy: 0.6949 Epoch 92/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7320 - accuracy: 0.7889 - val_loss: 1.5455 - val_accuracy: 0.6931 Epoch 93/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7282 - accuracy: 0.7909 - val_loss: 1.5532 - val_accuracy: 0.6930 Epoch 94/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7280 - accuracy: 0.7901 - val_loss: 1.5289 - val_accuracy: 0.6918 Epoch 95/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7307 - accuracy: 0.7891 - val_loss: 1.5392 - val_accuracy: 0.6943 Epoch 96/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7303 - accuracy: 0.7886 - val_loss: 1.5455 - val_accuracy: 0.6951 Epoch 97/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7257 - accuracy: 0.7909 - val_loss: 1.5460 - val_accuracy: 0.6932 Epoch 98/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7240 - accuracy: 0.7910 - val_loss: 1.5475 - val_accuracy: 0.6975 Epoch 99/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7240 - accuracy: 0.7899 - val_loss: 1.5562 - val_accuracy: 0.6948 Epoch 100/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7227 - accuracy: 0.7929 - val_loss: 1.5604 - val_accuracy: 0.6936
LSTM_V1.summary()
Model: "lstm_v1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
lstm (LSTM) (None, 256) 273408
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 593,541
Trainable params: 593,541
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(LSTM_V1_history.history)
Observations
LSTM_V1.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.5591 - accuracy: 0.6976
[1.5591022968292236, 0.6976135969161987]
Observations
tf.keras.backend.clear_session()
# Create the model
LSTM_V2 = Sequential(
name='lstm_v2',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
LSTM(64, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
LSTM_V2.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
LSTM_V2_history = LSTM_V2.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 7s 5ms/step - loss: 5.2943 - accuracy: 0.0977 - val_loss: 4.9838 - val_accuracy: 0.1261 Epoch 2/100 1148/1148 [==============================] - 5s 5ms/step - loss: 4.8138 - accuracy: 0.1377 - val_loss: 4.6586 - val_accuracy: 0.1477 Epoch 3/100 1148/1148 [==============================] - 5s 5ms/step - loss: 4.4596 - accuracy: 0.1678 - val_loss: 4.2644 - val_accuracy: 0.1903 Epoch 4/100 1148/1148 [==============================] - 5s 5ms/step - loss: 4.0955 - accuracy: 0.2142 - val_loss: 3.9578 - val_accuracy: 0.2346 Epoch 5/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.8209 - accuracy: 0.2485 - val_loss: 3.7048 - val_accuracy: 0.2727 Epoch 6/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.5928 - accuracy: 0.2794 - val_loss: 3.4992 - val_accuracy: 0.3048 Epoch 7/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.3989 - accuracy: 0.3072 - val_loss: 3.3239 - val_accuracy: 0.3288 Epoch 8/100 1148/1148 [==============================] - 5s 4ms/step - loss: 3.2329 - accuracy: 0.3250 - val_loss: 3.1712 - val_accuracy: 0.3505 Epoch 9/100 1148/1148 [==============================] - 5s 4ms/step - loss: 3.0891 - accuracy: 0.3460 - val_loss: 3.0358 - val_accuracy: 0.3777 Epoch 10/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.9635 - accuracy: 0.3644 - val_loss: 2.9170 - val_accuracy: 0.3965 Epoch 11/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.8458 - accuracy: 0.3821 - val_loss: 2.8086 - val_accuracy: 0.4130 Epoch 12/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.7458 - accuracy: 0.3986 - val_loss: 2.7132 - val_accuracy: 0.4283 Epoch 13/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.6555 - accuracy: 0.4108 - val_loss: 2.6286 - val_accuracy: 0.4400 Epoch 14/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.5720 - accuracy: 0.4261 - val_loss: 2.5563 - val_accuracy: 0.4557 Epoch 15/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.5041 - accuracy: 0.4352 - val_loss: 2.4836 - val_accuracy: 0.4674 Epoch 16/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.4298 - accuracy: 0.4481 - val_loss: 2.4215 - val_accuracy: 0.4779 Epoch 17/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.3661 - accuracy: 0.4578 - val_loss: 2.3607 - val_accuracy: 0.4895 Epoch 18/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.3094 - accuracy: 0.4675 - val_loss: 2.3147 - val_accuracy: 0.5012 Epoch 19/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.2561 - accuracy: 0.4759 - val_loss: 2.2598 - val_accuracy: 0.5145 Epoch 20/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.2099 - accuracy: 0.4845 - val_loss: 2.2126 - val_accuracy: 0.5189 Epoch 21/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.1603 - accuracy: 0.4940 - val_loss: 2.1804 - val_accuracy: 0.5229 Epoch 22/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.1251 - accuracy: 0.4984 - val_loss: 2.1388 - val_accuracy: 0.5288 Epoch 23/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.0761 - accuracy: 0.5098 - val_loss: 2.1021 - val_accuracy: 0.5341 Epoch 24/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.0423 - accuracy: 0.5123 - val_loss: 2.0681 - val_accuracy: 0.5426 Epoch 25/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.0108 - accuracy: 0.5199 - val_loss: 2.0390 - val_accuracy: 0.5467 Epoch 26/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.9725 - accuracy: 0.5306 - val_loss: 1.9997 - val_accuracy: 0.5559 Epoch 27/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.9488 - accuracy: 0.5328 - val_loss: 1.9762 - val_accuracy: 0.5579 Epoch 28/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.9156 - accuracy: 0.5367 - val_loss: 1.9546 - val_accuracy: 0.5584 Epoch 29/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.8834 - accuracy: 0.5418 - val_loss: 1.9297 - val_accuracy: 0.5655 Epoch 30/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.8692 - accuracy: 0.5451 - val_loss: 1.9010 - val_accuracy: 0.5729 Epoch 31/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.8359 - accuracy: 0.5522 - val_loss: 1.8867 - val_accuracy: 0.5763 Epoch 32/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.8195 - accuracy: 0.5562 - val_loss: 1.8667 - val_accuracy: 0.5763 Epoch 33/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.8007 - accuracy: 0.5584 - val_loss: 1.8419 - val_accuracy: 0.5834 Epoch 34/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.7752 - accuracy: 0.5646 - val_loss: 1.8240 - val_accuracy: 0.5906 Epoch 35/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7603 - accuracy: 0.5669 - val_loss: 1.8101 - val_accuracy: 0.5915 Epoch 36/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7380 - accuracy: 0.5715 - val_loss: 1.7895 - val_accuracy: 0.5944 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7169 - accuracy: 0.5766 - val_loss: 1.7736 - val_accuracy: 0.6004 Epoch 38/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6998 - accuracy: 0.5811 - val_loss: 1.7609 - val_accuracy: 0.6047 Epoch 39/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6918 - accuracy: 0.5803 - val_loss: 1.7475 - val_accuracy: 0.6081 Epoch 40/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6708 - accuracy: 0.5846 - val_loss: 1.7304 - val_accuracy: 0.6124 Epoch 41/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6609 - accuracy: 0.5862 - val_loss: 1.7252 - val_accuracy: 0.6110 Epoch 42/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6332 - accuracy: 0.5928 - val_loss: 1.7129 - val_accuracy: 0.6152 Epoch 43/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6262 - accuracy: 0.5945 - val_loss: 1.6945 - val_accuracy: 0.6178 Epoch 44/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6129 - accuracy: 0.5942 - val_loss: 1.6819 - val_accuracy: 0.6183 Epoch 45/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5971 - accuracy: 0.5986 - val_loss: 1.6801 - val_accuracy: 0.6229 Epoch 46/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5937 - accuracy: 0.5969 - val_loss: 1.6675 - val_accuracy: 0.6274 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5736 - accuracy: 0.6047 - val_loss: 1.6555 - val_accuracy: 0.6259 Epoch 48/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5624 - accuracy: 0.6047 - val_loss: 1.6450 - val_accuracy: 0.6333 Epoch 49/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5511 - accuracy: 0.6086 - val_loss: 1.6349 - val_accuracy: 0.6323 Epoch 50/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5432 - accuracy: 0.6109 - val_loss: 1.6320 - val_accuracy: 0.6310 Epoch 51/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5295 - accuracy: 0.6139 - val_loss: 1.6155 - val_accuracy: 0.6337 Epoch 52/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5183 - accuracy: 0.6141 - val_loss: 1.6167 - val_accuracy: 0.6349 Epoch 53/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5040 - accuracy: 0.6196 - val_loss: 1.6078 - val_accuracy: 0.6371 Epoch 54/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5013 - accuracy: 0.6196 - val_loss: 1.5961 - val_accuracy: 0.6369 Epoch 55/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.4862 - accuracy: 0.6202 - val_loss: 1.5985 - val_accuracy: 0.6402 Epoch 56/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4797 - accuracy: 0.6233 - val_loss: 1.5870 - val_accuracy: 0.6417 Epoch 57/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4643 - accuracy: 0.6272 - val_loss: 1.5738 - val_accuracy: 0.6465 Epoch 58/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4624 - accuracy: 0.6271 - val_loss: 1.5812 - val_accuracy: 0.6429 Epoch 59/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4536 - accuracy: 0.6296 - val_loss: 1.5637 - val_accuracy: 0.6465 Epoch 60/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4471 - accuracy: 0.6303 - val_loss: 1.5708 - val_accuracy: 0.6464 Epoch 61/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4366 - accuracy: 0.6307 - val_loss: 1.5638 - val_accuracy: 0.6482 Epoch 62/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4344 - accuracy: 0.6317 - val_loss: 1.5567 - val_accuracy: 0.6510 Epoch 63/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4208 - accuracy: 0.6361 - val_loss: 1.5480 - val_accuracy: 0.6498 Epoch 64/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4162 - accuracy: 0.6359 - val_loss: 1.5502 - val_accuracy: 0.6523 Epoch 65/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4120 - accuracy: 0.6386 - val_loss: 1.5404 - val_accuracy: 0.6550 Epoch 66/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4008 - accuracy: 0.6381 - val_loss: 1.5394 - val_accuracy: 0.6523 Epoch 67/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3989 - accuracy: 0.6410 - val_loss: 1.5217 - val_accuracy: 0.6535 Epoch 68/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3883 - accuracy: 0.6405 - val_loss: 1.5312 - val_accuracy: 0.6529 Epoch 69/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3826 - accuracy: 0.6430 - val_loss: 1.5230 - val_accuracy: 0.6564 Epoch 70/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3752 - accuracy: 0.6423 - val_loss: 1.5166 - val_accuracy: 0.6594 Epoch 71/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3760 - accuracy: 0.6426 - val_loss: 1.5053 - val_accuracy: 0.6554 Epoch 72/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3700 - accuracy: 0.6420 - val_loss: 1.5116 - val_accuracy: 0.6569 Epoch 73/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3608 - accuracy: 0.6494 - val_loss: 1.5083 - val_accuracy: 0.6596 Epoch 74/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3586 - accuracy: 0.6490 - val_loss: 1.5099 - val_accuracy: 0.6578 Epoch 75/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3538 - accuracy: 0.6502 - val_loss: 1.5036 - val_accuracy: 0.6607 Epoch 76/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3376 - accuracy: 0.6531 - val_loss: 1.5001 - val_accuracy: 0.6605 Epoch 77/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3378 - accuracy: 0.6540 - val_loss: 1.5036 - val_accuracy: 0.6617 Epoch 78/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3349 - accuracy: 0.6552 - val_loss: 1.4918 - val_accuracy: 0.6621 Epoch 79/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3354 - accuracy: 0.6519 - val_loss: 1.4960 - val_accuracy: 0.6640 Epoch 80/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3239 - accuracy: 0.6555 - val_loss: 1.4919 - val_accuracy: 0.6643 Epoch 81/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3207 - accuracy: 0.6589 - val_loss: 1.4902 - val_accuracy: 0.6652 Epoch 82/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3122 - accuracy: 0.6585 - val_loss: 1.4853 - val_accuracy: 0.6656 Epoch 83/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3105 - accuracy: 0.6584 - val_loss: 1.4780 - val_accuracy: 0.6682 Epoch 84/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3041 - accuracy: 0.6604 - val_loss: 1.4764 - val_accuracy: 0.6653 Epoch 85/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3020 - accuracy: 0.6591 - val_loss: 1.4733 - val_accuracy: 0.6671 Epoch 86/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2878 - accuracy: 0.6647 - val_loss: 1.4848 - val_accuracy: 0.6668 Epoch 87/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2971 - accuracy: 0.6608 - val_loss: 1.4688 - val_accuracy: 0.6675 Epoch 88/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2863 - accuracy: 0.6638 - val_loss: 1.4755 - val_accuracy: 0.6698 Epoch 89/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2905 - accuracy: 0.6621 - val_loss: 1.4730 - val_accuracy: 0.6684 Epoch 90/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2767 - accuracy: 0.6675 - val_loss: 1.4678 - val_accuracy: 0.6675 Epoch 91/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2769 - accuracy: 0.6655 - val_loss: 1.4654 - val_accuracy: 0.6672 Epoch 92/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2727 - accuracy: 0.6702 - val_loss: 1.4646 - val_accuracy: 0.6706 Epoch 93/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2690 - accuracy: 0.6655 - val_loss: 1.4603 - val_accuracy: 0.6741 Epoch 94/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2707 - accuracy: 0.6669 - val_loss: 1.4576 - val_accuracy: 0.6710 Epoch 95/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2621 - accuracy: 0.6678 - val_loss: 1.4589 - val_accuracy: 0.6723 Epoch 96/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2632 - accuracy: 0.6696 - val_loss: 1.4564 - val_accuracy: 0.6693 Epoch 97/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2585 - accuracy: 0.6695 - val_loss: 1.4557 - val_accuracy: 0.6724 Epoch 98/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2480 - accuracy: 0.6741 - val_loss: 1.4499 - val_accuracy: 0.6735 Epoch 99/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2528 - accuracy: 0.6713 - val_loss: 1.4639 - val_accuracy: 0.6720 Epoch 100/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2511 - accuracy: 0.6716 - val_loss: 1.4509 - val_accuracy: 0.6742
LSTM_V2.summary()
Model: "lstm_v2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
lstm (LSTM) (None, 64) 19200
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 109,125
Trainable params: 109,125
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(LSTM_V2_history.history)
Observations
LSTM_V2.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.4599 - accuracy: 0.6816
[1.4599246978759766, 0.6815952658653259]
Observations
tf.keras.backend.clear_session()
# Create the model
LSTM_V3 = Sequential(
name='lstm_v3',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
LSTM(128, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
LSTM_V3.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
LSTM_V3_history = LSTM_V3.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 7s 5ms/step - loss: 5.2243 - accuracy: 0.1087 - val_loss: 4.8404 - val_accuracy: 0.1299 Epoch 2/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.5886 - accuracy: 0.1574 - val_loss: 4.3372 - val_accuracy: 0.1859 Epoch 3/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.1104 - accuracy: 0.2161 - val_loss: 3.9402 - val_accuracy: 0.2401 Epoch 4/100 1148/1148 [==============================] - 7s 6ms/step - loss: 3.7337 - accuracy: 0.2597 - val_loss: 3.5937 - val_accuracy: 0.2817 Epoch 5/100 1148/1148 [==============================] - 7s 6ms/step - loss: 3.4169 - accuracy: 0.2985 - val_loss: 3.3127 - val_accuracy: 0.3197 Epoch 6/100 1148/1148 [==============================] - 7s 6ms/step - loss: 3.1360 - accuracy: 0.3376 - val_loss: 3.0525 - val_accuracy: 0.3696 Epoch 7/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.8905 - accuracy: 0.3757 - val_loss: 2.8428 - val_accuracy: 0.3994 Epoch 8/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6880 - accuracy: 0.4064 - val_loss: 2.6626 - val_accuracy: 0.4355 Epoch 9/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5110 - accuracy: 0.4364 - val_loss: 2.5027 - val_accuracy: 0.4625 Epoch 10/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3585 - accuracy: 0.4626 - val_loss: 2.3772 - val_accuracy: 0.4877 Epoch 11/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2321 - accuracy: 0.4839 - val_loss: 2.2659 - val_accuracy: 0.5076 Epoch 12/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1242 - accuracy: 0.5040 - val_loss: 2.1709 - val_accuracy: 0.5246 Epoch 13/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0223 - accuracy: 0.5253 - val_loss: 2.0841 - val_accuracy: 0.5354 Epoch 14/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9315 - accuracy: 0.5395 - val_loss: 2.0075 - val_accuracy: 0.5522 Epoch 15/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8575 - accuracy: 0.5553 - val_loss: 1.9440 - val_accuracy: 0.5691 Epoch 16/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.7875 - accuracy: 0.5663 - val_loss: 1.8932 - val_accuracy: 0.5767 Epoch 17/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.7245 - accuracy: 0.5810 - val_loss: 1.8361 - val_accuracy: 0.5919 Epoch 18/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6691 - accuracy: 0.5914 - val_loss: 1.7954 - val_accuracy: 0.5974 Epoch 19/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6150 - accuracy: 0.6024 - val_loss: 1.7485 - val_accuracy: 0.6060 Epoch 20/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5768 - accuracy: 0.6103 - val_loss: 1.7186 - val_accuracy: 0.6140 Epoch 21/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5376 - accuracy: 0.6194 - val_loss: 1.6829 - val_accuracy: 0.6199 Epoch 22/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4951 - accuracy: 0.6271 - val_loss: 1.6522 - val_accuracy: 0.6261 Epoch 23/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4545 - accuracy: 0.6346 - val_loss: 1.6228 - val_accuracy: 0.6315 Epoch 24/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4240 - accuracy: 0.6419 - val_loss: 1.6004 - val_accuracy: 0.6355 Epoch 25/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3965 - accuracy: 0.6454 - val_loss: 1.5809 - val_accuracy: 0.6421 Epoch 26/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3673 - accuracy: 0.6530 - val_loss: 1.5639 - val_accuracy: 0.6465 Epoch 27/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3536 - accuracy: 0.6560 - val_loss: 1.5421 - val_accuracy: 0.6483 Epoch 28/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3251 - accuracy: 0.6614 - val_loss: 1.5304 - val_accuracy: 0.6530 Epoch 29/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3001 - accuracy: 0.6676 - val_loss: 1.5185 - val_accuracy: 0.6547 Epoch 30/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2812 - accuracy: 0.6710 - val_loss: 1.5003 - val_accuracy: 0.6558 Epoch 31/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2577 - accuracy: 0.6757 - val_loss: 1.4924 - val_accuracy: 0.6603 Epoch 32/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2369 - accuracy: 0.6828 - val_loss: 1.4709 - val_accuracy: 0.6657 Epoch 33/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2211 - accuracy: 0.6854 - val_loss: 1.4706 - val_accuracy: 0.6652 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2072 - accuracy: 0.6869 - val_loss: 1.4651 - val_accuracy: 0.6670 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1965 - accuracy: 0.6905 - val_loss: 1.4473 - val_accuracy: 0.6672 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1823 - accuracy: 0.6938 - val_loss: 1.4349 - val_accuracy: 0.6729 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1641 - accuracy: 0.6959 - val_loss: 1.4438 - val_accuracy: 0.6744 Epoch 38/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1510 - accuracy: 0.7021 - val_loss: 1.4260 - val_accuracy: 0.6717 Epoch 39/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1433 - accuracy: 0.7016 - val_loss: 1.4209 - val_accuracy: 0.6766 Epoch 40/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1222 - accuracy: 0.7049 - val_loss: 1.4148 - val_accuracy: 0.6764 Epoch 41/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1201 - accuracy: 0.7061 - val_loss: 1.4153 - val_accuracy: 0.6778 Epoch 42/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1062 - accuracy: 0.7087 - val_loss: 1.4020 - val_accuracy: 0.6805 Epoch 43/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0976 - accuracy: 0.7102 - val_loss: 1.4087 - val_accuracy: 0.6788 Epoch 44/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0923 - accuracy: 0.7112 - val_loss: 1.3996 - val_accuracy: 0.6844 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0830 - accuracy: 0.7163 - val_loss: 1.4044 - val_accuracy: 0.6831 Epoch 46/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0707 - accuracy: 0.7154 - val_loss: 1.3970 - val_accuracy: 0.6836 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0668 - accuracy: 0.7178 - val_loss: 1.3930 - val_accuracy: 0.6861 Epoch 48/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0594 - accuracy: 0.7173 - val_loss: 1.3892 - val_accuracy: 0.6862 Epoch 49/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0501 - accuracy: 0.7203 - val_loss: 1.3836 - val_accuracy: 0.6869 Epoch 50/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0480 - accuracy: 0.7196 - val_loss: 1.3891 - val_accuracy: 0.6856 Epoch 51/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0361 - accuracy: 0.7245 - val_loss: 1.3829 - val_accuracy: 0.6883 Epoch 52/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0219 - accuracy: 0.7278 - val_loss: 1.3840 - val_accuracy: 0.6900 Epoch 53/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0176 - accuracy: 0.7300 - val_loss: 1.3792 - val_accuracy: 0.6924 Epoch 54/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0135 - accuracy: 0.7293 - val_loss: 1.3740 - val_accuracy: 0.6880 Epoch 55/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0109 - accuracy: 0.7306 - val_loss: 1.3708 - val_accuracy: 0.6894 Epoch 56/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0103 - accuracy: 0.7276 - val_loss: 1.3798 - val_accuracy: 0.6897 Epoch 57/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0007 - accuracy: 0.7317 - val_loss: 1.3688 - val_accuracy: 0.6906 Epoch 58/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9940 - accuracy: 0.7321 - val_loss: 1.3783 - val_accuracy: 0.6920 Epoch 59/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9916 - accuracy: 0.7336 - val_loss: 1.3700 - val_accuracy: 0.6933 Epoch 60/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9816 - accuracy: 0.7334 - val_loss: 1.3735 - val_accuracy: 0.6931 Epoch 61/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9757 - accuracy: 0.7373 - val_loss: 1.3736 - val_accuracy: 0.6948 Epoch 62/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9688 - accuracy: 0.7387 - val_loss: 1.3676 - val_accuracy: 0.6951 Epoch 63/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9669 - accuracy: 0.7391 - val_loss: 1.3784 - val_accuracy: 0.6934 Epoch 64/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9687 - accuracy: 0.7397 - val_loss: 1.3669 - val_accuracy: 0.6942 Epoch 65/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9609 - accuracy: 0.7412 - val_loss: 1.3729 - val_accuracy: 0.6964 Epoch 66/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9504 - accuracy: 0.7420 - val_loss: 1.3601 - val_accuracy: 0.6955 Epoch 67/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9576 - accuracy: 0.7401 - val_loss: 1.3727 - val_accuracy: 0.6963 Epoch 68/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9444 - accuracy: 0.7441 - val_loss: 1.3769 - val_accuracy: 0.6949 Epoch 69/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9449 - accuracy: 0.7424 - val_loss: 1.3723 - val_accuracy: 0.6950 Epoch 70/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9395 - accuracy: 0.7448 - val_loss: 1.3688 - val_accuracy: 0.6967 Epoch 71/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9380 - accuracy: 0.7438 - val_loss: 1.3728 - val_accuracy: 0.6974 Epoch 72/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9334 - accuracy: 0.7475 - val_loss: 1.3753 - val_accuracy: 0.6971 Epoch 73/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9327 - accuracy: 0.7458 - val_loss: 1.3668 - val_accuracy: 0.6992 Epoch 74/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9276 - accuracy: 0.7459 - val_loss: 1.3730 - val_accuracy: 0.6984 Epoch 75/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9191 - accuracy: 0.7500 - val_loss: 1.3744 - val_accuracy: 0.6949 Epoch 76/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9170 - accuracy: 0.7506 - val_loss: 1.3682 - val_accuracy: 0.6941 Epoch 77/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9133 - accuracy: 0.7494 - val_loss: 1.3700 - val_accuracy: 0.6978 Epoch 78/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9117 - accuracy: 0.7503 - val_loss: 1.3761 - val_accuracy: 0.6981 Epoch 79/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9162 - accuracy: 0.7478 - val_loss: 1.3799 - val_accuracy: 0.6989 Epoch 80/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9105 - accuracy: 0.7489 - val_loss: 1.3698 - val_accuracy: 0.6987 Epoch 81/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9026 - accuracy: 0.7537 - val_loss: 1.3785 - val_accuracy: 0.6975 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9079 - accuracy: 0.7497 - val_loss: 1.3850 - val_accuracy: 0.6961 Epoch 83/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9059 - accuracy: 0.7505 - val_loss: 1.3721 - val_accuracy: 0.6959 Epoch 84/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9003 - accuracy: 0.7518 - val_loss: 1.3821 - val_accuracy: 0.6978 Epoch 85/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8956 - accuracy: 0.7552 - val_loss: 1.3702 - val_accuracy: 0.6980 Epoch 86/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8967 - accuracy: 0.7517 - val_loss: 1.3842 - val_accuracy: 0.6978 Epoch 87/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8901 - accuracy: 0.7545 - val_loss: 1.3765 - val_accuracy: 0.7002 Epoch 88/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8858 - accuracy: 0.7581 - val_loss: 1.3806 - val_accuracy: 0.6983 Epoch 89/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8843 - accuracy: 0.7565 - val_loss: 1.3844 - val_accuracy: 0.7006 Epoch 90/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8883 - accuracy: 0.7557 - val_loss: 1.3932 - val_accuracy: 0.6979 Epoch 91/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8807 - accuracy: 0.7572 - val_loss: 1.3832 - val_accuracy: 0.7010 Epoch 92/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8786 - accuracy: 0.7578 - val_loss: 1.3931 - val_accuracy: 0.6989 Epoch 93/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8810 - accuracy: 0.7570 - val_loss: 1.3940 - val_accuracy: 0.6989 Epoch 94/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8799 - accuracy: 0.7560 - val_loss: 1.3925 - val_accuracy: 0.6957 Epoch 95/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8797 - accuracy: 0.7576 - val_loss: 1.3918 - val_accuracy: 0.6973 Epoch 96/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8707 - accuracy: 0.7597 - val_loss: 1.3925 - val_accuracy: 0.7000 Epoch 97/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8713 - accuracy: 0.7609 - val_loss: 1.3959 - val_accuracy: 0.6977 Epoch 98/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8673 - accuracy: 0.7592 - val_loss: 1.3963 - val_accuracy: 0.7010 Epoch 99/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8736 - accuracy: 0.7572 - val_loss: 1.3919 - val_accuracy: 0.7001 Epoch 100/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8650 - accuracy: 0.7612 - val_loss: 1.3951 - val_accuracy: 0.6987
LSTM_V3.summary()
Model: "lstm_v3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
lstm (LSTM) (None, 128) 71168
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 237,829
Trainable params: 237,829
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(LSTM_V3_history.history)
Observations
LSTM_V3.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.3818 - accuracy: 0.7010
[1.3817840814590454, 0.7009643912315369]
Observations
tf.keras.backend.clear_session()
# Create the model
LSTM_V4 = Sequential(
name='lstm_v4',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
LSTM(128, activation='tanh', return_sequences=True),
LSTM(64, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
LSTM_V4.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
LSTM_V4_history = LSTM_V4.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 10s 8ms/step - loss: 5.2703 - accuracy: 0.1010 - val_loss: 4.8707 - val_accuracy: 0.1286 Epoch 2/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.6822 - accuracy: 0.1454 - val_loss: 4.5236 - val_accuracy: 0.1646 Epoch 3/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.7848 - accuracy: 0.1403 - val_loss: 4.7249 - val_accuracy: 0.1387 Epoch 4/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.5349 - accuracy: 0.1544 - val_loss: 4.3557 - val_accuracy: 0.1765 Epoch 5/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.2777 - accuracy: 0.1833 - val_loss: 4.1659 - val_accuracy: 0.2038 Epoch 6/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.0831 - accuracy: 0.2067 - val_loss: 3.9639 - val_accuracy: 0.2400 Epoch 7/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.8647 - accuracy: 0.2326 - val_loss: 3.7475 - val_accuracy: 0.2593 Epoch 8/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.6603 - accuracy: 0.2563 - val_loss: 3.5520 - val_accuracy: 0.2821 Epoch 9/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.4805 - accuracy: 0.2777 - val_loss: 3.3911 - val_accuracy: 0.3017 Epoch 10/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.3337 - accuracy: 0.2972 - val_loss: 3.2568 - val_accuracy: 0.3216 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.2009 - accuracy: 0.3137 - val_loss: 3.1388 - val_accuracy: 0.3393 Epoch 12/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.0965 - accuracy: 0.3279 - val_loss: 3.0485 - val_accuracy: 0.3568 Epoch 13/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.0011 - accuracy: 0.3413 - val_loss: 2.9625 - val_accuracy: 0.3769 Epoch 14/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.9168 - accuracy: 0.3525 - val_loss: 2.8903 - val_accuracy: 0.3839 Epoch 15/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.8400 - accuracy: 0.3636 - val_loss: 2.8244 - val_accuracy: 0.3939 Epoch 16/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.7699 - accuracy: 0.3743 - val_loss: 2.7472 - val_accuracy: 0.4116 Epoch 17/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.7061 - accuracy: 0.3841 - val_loss: 2.6912 - val_accuracy: 0.4167 Epoch 18/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.6510 - accuracy: 0.3968 - val_loss: 2.6322 - val_accuracy: 0.4313 Epoch 19/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.5865 - accuracy: 0.4061 - val_loss: 2.5800 - val_accuracy: 0.4400 Epoch 20/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.5304 - accuracy: 0.4138 - val_loss: 2.5330 - val_accuracy: 0.4492 Epoch 21/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.4803 - accuracy: 0.4256 - val_loss: 2.4842 - val_accuracy: 0.4564 Epoch 22/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.4411 - accuracy: 0.4297 - val_loss: 2.4379 - val_accuracy: 0.4662 Epoch 23/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.3893 - accuracy: 0.4413 - val_loss: 2.4126 - val_accuracy: 0.4749 Epoch 24/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3523 - accuracy: 0.4467 - val_loss: 2.3723 - val_accuracy: 0.4807 Epoch 25/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3176 - accuracy: 0.4513 - val_loss: 2.3291 - val_accuracy: 0.4894 Epoch 26/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.2771 - accuracy: 0.4603 - val_loss: 2.3023 - val_accuracy: 0.4925 Epoch 27/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.2351 - accuracy: 0.4684 - val_loss: 2.2634 - val_accuracy: 0.5044 Epoch 28/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.2003 - accuracy: 0.4776 - val_loss: 2.2354 - val_accuracy: 0.5069 Epoch 29/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1734 - accuracy: 0.4805 - val_loss: 2.2108 - val_accuracy: 0.5120 Epoch 30/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1366 - accuracy: 0.4885 - val_loss: 2.1818 - val_accuracy: 0.5193 Epoch 31/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.1045 - accuracy: 0.4922 - val_loss: 2.1467 - val_accuracy: 0.5257 Epoch 32/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.0727 - accuracy: 0.5009 - val_loss: 2.1217 - val_accuracy: 0.5320 Epoch 33/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.0456 - accuracy: 0.5028 - val_loss: 2.1043 - val_accuracy: 0.5372 Epoch 34/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.0169 - accuracy: 0.5140 - val_loss: 2.0834 - val_accuracy: 0.5416 Epoch 35/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.9948 - accuracy: 0.5160 - val_loss: 2.0639 - val_accuracy: 0.5463 Epoch 36/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.9688 - accuracy: 0.5205 - val_loss: 2.0498 - val_accuracy: 0.5463 Epoch 37/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9533 - accuracy: 0.5239 - val_loss: 2.0128 - val_accuracy: 0.5568 Epoch 38/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9234 - accuracy: 0.5304 - val_loss: 2.0015 - val_accuracy: 0.5563 Epoch 39/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9082 - accuracy: 0.5320 - val_loss: 1.9826 - val_accuracy: 0.5637 Epoch 40/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8799 - accuracy: 0.5399 - val_loss: 1.9578 - val_accuracy: 0.5682 Epoch 41/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8549 - accuracy: 0.5447 - val_loss: 1.9406 - val_accuracy: 0.5697 Epoch 42/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8341 - accuracy: 0.5476 - val_loss: 1.9225 - val_accuracy: 0.5780 Epoch 43/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8161 - accuracy: 0.5500 - val_loss: 1.9124 - val_accuracy: 0.5803 Epoch 44/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7968 - accuracy: 0.5566 - val_loss: 1.8899 - val_accuracy: 0.5834 Epoch 45/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7728 - accuracy: 0.5621 - val_loss: 1.8732 - val_accuracy: 0.5891 Epoch 46/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7628 - accuracy: 0.5643 - val_loss: 1.8686 - val_accuracy: 0.5896 Epoch 47/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7382 - accuracy: 0.5696 - val_loss: 1.8502 - val_accuracy: 0.5947 Epoch 48/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7268 - accuracy: 0.5713 - val_loss: 1.8369 - val_accuracy: 0.5949 Epoch 49/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7054 - accuracy: 0.5767 - val_loss: 1.8261 - val_accuracy: 0.6005 Epoch 50/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6919 - accuracy: 0.5779 - val_loss: 1.8183 - val_accuracy: 0.6025 Epoch 51/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6796 - accuracy: 0.5819 - val_loss: 1.7913 - val_accuracy: 0.6062 Epoch 52/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6693 - accuracy: 0.5855 - val_loss: 1.7858 - val_accuracy: 0.6067 Epoch 53/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6470 - accuracy: 0.5923 - val_loss: 1.7887 - val_accuracy: 0.6052 Epoch 54/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6384 - accuracy: 0.5924 - val_loss: 1.7608 - val_accuracy: 0.6092 Epoch 55/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6108 - accuracy: 0.5979 - val_loss: 1.7519 - val_accuracy: 0.6156 Epoch 56/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6020 - accuracy: 0.5971 - val_loss: 1.7403 - val_accuracy: 0.6132 Epoch 57/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5894 - accuracy: 0.6004 - val_loss: 1.7371 - val_accuracy: 0.6157 Epoch 58/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5825 - accuracy: 0.6030 - val_loss: 1.7300 - val_accuracy: 0.6168 Epoch 59/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5745 - accuracy: 0.6055 - val_loss: 1.7110 - val_accuracy: 0.6215 Epoch 60/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5521 - accuracy: 0.6106 - val_loss: 1.7100 - val_accuracy: 0.6223 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5427 - accuracy: 0.6127 - val_loss: 1.7021 - val_accuracy: 0.6239 Epoch 62/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5273 - accuracy: 0.6148 - val_loss: 1.7090 - val_accuracy: 0.6246 Epoch 63/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5175 - accuracy: 0.6153 - val_loss: 1.7014 - val_accuracy: 0.6275 Epoch 64/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5077 - accuracy: 0.6188 - val_loss: 1.6917 - val_accuracy: 0.6278 Epoch 65/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4957 - accuracy: 0.6210 - val_loss: 1.6786 - val_accuracy: 0.6313 Epoch 66/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.4854 - accuracy: 0.6232 - val_loss: 1.6781 - val_accuracy: 0.6300 Epoch 67/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4796 - accuracy: 0.6245 - val_loss: 1.6623 - val_accuracy: 0.6362 Epoch 68/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4906 - accuracy: 0.6260 - val_loss: 1.6758 - val_accuracy: 0.6375 Epoch 69/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.6451 - accuracy: 0.5276 - val_loss: 3.8408 - val_accuracy: 0.2360 Epoch 70/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.5551 - accuracy: 0.4114 - val_loss: 1.7779 - val_accuracy: 0.6065 Epoch 71/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5123 - accuracy: 0.6144 - val_loss: 1.6631 - val_accuracy: 0.6346 Epoch 72/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4471 - accuracy: 0.6323 - val_loss: 1.6544 - val_accuracy: 0.6374 Epoch 73/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4428 - accuracy: 0.6329 - val_loss: 1.6396 - val_accuracy: 0.6364 Epoch 74/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4230 - accuracy: 0.6368 - val_loss: 1.6281 - val_accuracy: 0.6393 Epoch 75/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4227 - accuracy: 0.6374 - val_loss: 1.6368 - val_accuracy: 0.6395 Epoch 76/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4183 - accuracy: 0.6381 - val_loss: 1.6353 - val_accuracy: 0.6425 Epoch 77/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4122 - accuracy: 0.6406 - val_loss: 1.6221 - val_accuracy: 0.6441 Epoch 78/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.4046 - accuracy: 0.6435 - val_loss: 1.6118 - val_accuracy: 0.6453 Epoch 79/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.3831 - accuracy: 0.6471 - val_loss: 1.6158 - val_accuracy: 0.6469 Epoch 80/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3832 - accuracy: 0.6471 - val_loss: 1.6060 - val_accuracy: 0.6499 Epoch 81/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.3796 - accuracy: 0.6466 - val_loss: 1.6103 - val_accuracy: 0.6465 Epoch 82/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3686 - accuracy: 0.6528 - val_loss: 1.5974 - val_accuracy: 0.6500 Epoch 83/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3568 - accuracy: 0.6524 - val_loss: 1.5918 - val_accuracy: 0.6517 Epoch 84/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3451 - accuracy: 0.6563 - val_loss: 1.5809 - val_accuracy: 0.6509 Epoch 85/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3467 - accuracy: 0.6557 - val_loss: 1.5900 - val_accuracy: 0.6505 Epoch 86/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3396 - accuracy: 0.6582 - val_loss: 1.5763 - val_accuracy: 0.6581 Epoch 87/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3265 - accuracy: 0.6579 - val_loss: 1.5776 - val_accuracy: 0.6556 Epoch 88/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3194 - accuracy: 0.6609 - val_loss: 1.5805 - val_accuracy: 0.6571 Epoch 89/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.3173 - accuracy: 0.6597 - val_loss: 1.5807 - val_accuracy: 0.6537 Epoch 90/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3114 - accuracy: 0.6622 - val_loss: 1.5685 - val_accuracy: 0.6573 Epoch 91/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3081 - accuracy: 0.6639 - val_loss: 1.5729 - val_accuracy: 0.6566 Epoch 92/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2950 - accuracy: 0.6681 - val_loss: 1.5627 - val_accuracy: 0.6570 Epoch 93/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2922 - accuracy: 0.6652 - val_loss: 1.5654 - val_accuracy: 0.6573 Epoch 94/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2804 - accuracy: 0.6698 - val_loss: 1.5595 - val_accuracy: 0.6621 Epoch 95/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2777 - accuracy: 0.6686 - val_loss: 1.5561 - val_accuracy: 0.6631 Epoch 96/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2730 - accuracy: 0.6709 - val_loss: 1.5608 - val_accuracy: 0.6596 Epoch 97/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2640 - accuracy: 0.6722 - val_loss: 1.5531 - val_accuracy: 0.6623 Epoch 98/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2655 - accuracy: 0.6732 - val_loss: 1.5555 - val_accuracy: 0.6607 Epoch 99/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2589 - accuracy: 0.6739 - val_loss: 1.5563 - val_accuracy: 0.6618 Epoch 100/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2531 - accuracy: 0.6748 - val_loss: 1.5409 - val_accuracy: 0.6656
LSTM_V4.summary()
Model: "lstm_v4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
lstm (LSTM) (None, 34, 128) 71168
lstm_1 (LSTM) (None, 64) 49408
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 210,501
Trainable params: 210,501
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(LSTM_V4_history.history)
Observations
LSTM_V4.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.5174 - accuracy: 0.6688
[1.5174171924591064, 0.6687642931938171]
Observations

Image Source: Dishashree, G., 2020
tf.keras.backend.clear_session()
# Create the model
GRU_V1 = Sequential(
name='gru_v1',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(128, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_V1.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_V1_history = GRU_V1.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 7s 5ms/step - loss: 5.0105 - accuracy: 0.1295 - val_loss: 4.3266 - val_accuracy: 0.1957 Epoch 2/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8680 - accuracy: 0.2475 - val_loss: 3.4879 - val_accuracy: 0.3041 Epoch 3/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.1805 - accuracy: 0.3371 - val_loss: 2.9429 - val_accuracy: 0.3915 Epoch 4/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.7148 - accuracy: 0.4115 - val_loss: 2.5883 - val_accuracy: 0.4515 Epoch 5/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3945 - accuracy: 0.4687 - val_loss: 2.3481 - val_accuracy: 0.5015 Epoch 6/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.1639 - accuracy: 0.5110 - val_loss: 2.1634 - val_accuracy: 0.5319 Epoch 7/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.9882 - accuracy: 0.5406 - val_loss: 2.0197 - val_accuracy: 0.5547 Epoch 8/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.8537 - accuracy: 0.5705 - val_loss: 1.9117 - val_accuracy: 0.5812 Epoch 9/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7516 - accuracy: 0.5896 - val_loss: 1.8203 - val_accuracy: 0.5987 Epoch 10/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6530 - accuracy: 0.6116 - val_loss: 1.7487 - val_accuracy: 0.6127 Epoch 11/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5819 - accuracy: 0.6242 - val_loss: 1.6955 - val_accuracy: 0.6241 Epoch 12/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5181 - accuracy: 0.6387 - val_loss: 1.6482 - val_accuracy: 0.6311 Epoch 13/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4596 - accuracy: 0.6494 - val_loss: 1.6016 - val_accuracy: 0.6454 Epoch 14/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4187 - accuracy: 0.6585 - val_loss: 1.5698 - val_accuracy: 0.6466 Epoch 15/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3850 - accuracy: 0.6641 - val_loss: 1.5478 - val_accuracy: 0.6548 Epoch 16/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3371 - accuracy: 0.6753 - val_loss: 1.5224 - val_accuracy: 0.6555 Epoch 17/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3009 - accuracy: 0.6816 - val_loss: 1.5032 - val_accuracy: 0.6632 Epoch 18/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2728 - accuracy: 0.6889 - val_loss: 1.4852 - val_accuracy: 0.6623 Epoch 19/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2505 - accuracy: 0.6920 - val_loss: 1.4706 - val_accuracy: 0.6657 Epoch 20/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2267 - accuracy: 0.6990 - val_loss: 1.4536 - val_accuracy: 0.6715 Epoch 21/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2062 - accuracy: 0.7028 - val_loss: 1.4554 - val_accuracy: 0.6714 Epoch 22/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1876 - accuracy: 0.7045 - val_loss: 1.4336 - val_accuracy: 0.6729 Epoch 23/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1678 - accuracy: 0.7105 - val_loss: 1.4256 - val_accuracy: 0.6767 Epoch 24/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1485 - accuracy: 0.7142 - val_loss: 1.4055 - val_accuracy: 0.6864 Epoch 25/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1319 - accuracy: 0.7169 - val_loss: 1.3996 - val_accuracy: 0.6847 Epoch 26/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1303 - accuracy: 0.7192 - val_loss: 1.3874 - val_accuracy: 0.6867 Epoch 27/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1121 - accuracy: 0.7214 - val_loss: 1.3958 - val_accuracy: 0.6828 Epoch 28/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1032 - accuracy: 0.7233 - val_loss: 1.3955 - val_accuracy: 0.6858 Epoch 29/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.0865 - accuracy: 0.7271 - val_loss: 1.3795 - val_accuracy: 0.6831 Epoch 30/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0801 - accuracy: 0.7284 - val_loss: 1.3745 - val_accuracy: 0.6876 Epoch 31/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0670 - accuracy: 0.7328 - val_loss: 1.3748 - val_accuracy: 0.6890 Epoch 32/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0692 - accuracy: 0.7315 - val_loss: 1.3690 - val_accuracy: 0.6898 Epoch 33/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0517 - accuracy: 0.7354 - val_loss: 1.3669 - val_accuracy: 0.6871 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0404 - accuracy: 0.7384 - val_loss: 1.3594 - val_accuracy: 0.6912 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0412 - accuracy: 0.7355 - val_loss: 1.3608 - val_accuracy: 0.6934 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0280 - accuracy: 0.7386 - val_loss: 1.3621 - val_accuracy: 0.6916 Epoch 37/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.0300 - accuracy: 0.7397 - val_loss: 1.3512 - val_accuracy: 0.6934 Epoch 38/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.0151 - accuracy: 0.7432 - val_loss: 1.3579 - val_accuracy: 0.6930 Epoch 39/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0115 - accuracy: 0.7425 - val_loss: 1.3486 - val_accuracy: 0.6959 Epoch 40/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.0046 - accuracy: 0.7429 - val_loss: 1.3567 - val_accuracy: 0.6957 Epoch 41/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.0014 - accuracy: 0.7453 - val_loss: 1.3493 - val_accuracy: 0.6957 Epoch 42/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9968 - accuracy: 0.7460 - val_loss: 1.3436 - val_accuracy: 0.6975 Epoch 43/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9876 - accuracy: 0.7474 - val_loss: 1.3561 - val_accuracy: 0.6918 Epoch 44/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9898 - accuracy: 0.7471 - val_loss: 1.3470 - val_accuracy: 0.6926 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9867 - accuracy: 0.7488 - val_loss: 1.3415 - val_accuracy: 0.6936 Epoch 46/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9826 - accuracy: 0.7469 - val_loss: 1.3425 - val_accuracy: 0.6979 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9758 - accuracy: 0.7490 - val_loss: 1.3397 - val_accuracy: 0.6973 Epoch 48/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9740 - accuracy: 0.7518 - val_loss: 1.3375 - val_accuracy: 0.6955 Epoch 49/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9709 - accuracy: 0.7502 - val_loss: 1.3431 - val_accuracy: 0.6974 Epoch 50/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9670 - accuracy: 0.7530 - val_loss: 1.3324 - val_accuracy: 0.6972 Epoch 51/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9638 - accuracy: 0.7519 - val_loss: 1.3471 - val_accuracy: 0.6994 Epoch 52/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9566 - accuracy: 0.7527 - val_loss: 1.3341 - val_accuracy: 0.7016 Epoch 53/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9664 - accuracy: 0.7517 - val_loss: 1.3437 - val_accuracy: 0.6986 Epoch 54/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9518 - accuracy: 0.7549 - val_loss: 1.3437 - val_accuracy: 0.7006 Epoch 55/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9449 - accuracy: 0.7558 - val_loss: 1.3321 - val_accuracy: 0.6983 Epoch 56/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9515 - accuracy: 0.7551 - val_loss: 1.3501 - val_accuracy: 0.6962 Epoch 57/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9441 - accuracy: 0.7564 - val_loss: 1.3378 - val_accuracy: 0.7033 Epoch 58/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9455 - accuracy: 0.7555 - val_loss: 1.3320 - val_accuracy: 0.7012 Epoch 59/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9363 - accuracy: 0.7569 - val_loss: 1.3401 - val_accuracy: 0.6995 Epoch 60/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9378 - accuracy: 0.7567 - val_loss: 1.3464 - val_accuracy: 0.6984 Epoch 61/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9324 - accuracy: 0.7578 - val_loss: 1.3360 - val_accuracy: 0.6992 Epoch 62/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9329 - accuracy: 0.7602 - val_loss: 1.3386 - val_accuracy: 0.7027 Epoch 63/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9312 - accuracy: 0.7593 - val_loss: 1.3466 - val_accuracy: 0.6984 Epoch 64/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9224 - accuracy: 0.7614 - val_loss: 1.3345 - val_accuracy: 0.6990 Epoch 65/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9306 - accuracy: 0.7589 - val_loss: 1.3450 - val_accuracy: 0.6997 Epoch 66/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.9244 - accuracy: 0.7586 - val_loss: 1.3384 - val_accuracy: 0.7018 Epoch 67/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9237 - accuracy: 0.7622 - val_loss: 1.3409 - val_accuracy: 0.6996 Epoch 68/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9237 - accuracy: 0.7574 - val_loss: 1.3423 - val_accuracy: 0.7011 Epoch 69/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9220 - accuracy: 0.7609 - val_loss: 1.3529 - val_accuracy: 0.7005 Epoch 70/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9192 - accuracy: 0.7629 - val_loss: 1.3452 - val_accuracy: 0.6988 Epoch 71/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9152 - accuracy: 0.7621 - val_loss: 1.3403 - val_accuracy: 0.6989 Epoch 72/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9157 - accuracy: 0.7620 - val_loss: 1.3455 - val_accuracy: 0.7024 Epoch 73/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9093 - accuracy: 0.7648 - val_loss: 1.3386 - val_accuracy: 0.7055 Epoch 74/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9096 - accuracy: 0.7592 - val_loss: 1.3407 - val_accuracy: 0.7005 Epoch 75/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9075 - accuracy: 0.7647 - val_loss: 1.3526 - val_accuracy: 0.6985 Epoch 76/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9054 - accuracy: 0.7648 - val_loss: 1.3427 - val_accuracy: 0.6983 Epoch 77/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9041 - accuracy: 0.7625 - val_loss: 1.3493 - val_accuracy: 0.6979 Epoch 78/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9053 - accuracy: 0.7637 - val_loss: 1.3345 - val_accuracy: 0.7036 Epoch 79/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9052 - accuracy: 0.7635 - val_loss: 1.3395 - val_accuracy: 0.7014 Epoch 80/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9023 - accuracy: 0.7650 - val_loss: 1.3401 - val_accuracy: 0.7017 Epoch 81/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8954 - accuracy: 0.7661 - val_loss: 1.3481 - val_accuracy: 0.6974 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8976 - accuracy: 0.7672 - val_loss: 1.3366 - val_accuracy: 0.7010 Epoch 83/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8907 - accuracy: 0.7678 - val_loss: 1.3545 - val_accuracy: 0.7024 Epoch 84/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8931 - accuracy: 0.7674 - val_loss: 1.3476 - val_accuracy: 0.6989 Epoch 85/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8916 - accuracy: 0.7639 - val_loss: 1.3323 - val_accuracy: 0.6979 Epoch 86/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8937 - accuracy: 0.7663 - val_loss: 1.3527 - val_accuracy: 0.7026 Epoch 87/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8865 - accuracy: 0.7676 - val_loss: 1.3537 - val_accuracy: 0.7027 Epoch 88/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8962 - accuracy: 0.7661 - val_loss: 1.3556 - val_accuracy: 0.6991 Epoch 89/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8896 - accuracy: 0.7666 - val_loss: 1.3523 - val_accuracy: 0.7017 Epoch 90/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8885 - accuracy: 0.7688 - val_loss: 1.3427 - val_accuracy: 0.6998 Epoch 91/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8827 - accuracy: 0.7685 - val_loss: 1.3479 - val_accuracy: 0.7019 Epoch 92/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8806 - accuracy: 0.7693 - val_loss: 1.3446 - val_accuracy: 0.7010 Epoch 93/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8883 - accuracy: 0.7679 - val_loss: 1.3510 - val_accuracy: 0.7004 Epoch 94/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8836 - accuracy: 0.7669 - val_loss: 1.3488 - val_accuracy: 0.7000 Epoch 95/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8889 - accuracy: 0.7672 - val_loss: 1.3435 - val_accuracy: 0.7019 Epoch 96/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8789 - accuracy: 0.7677 - val_loss: 1.3478 - val_accuracy: 0.7036 Epoch 97/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8788 - accuracy: 0.7689 - val_loss: 1.3508 - val_accuracy: 0.7010 Epoch 98/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8816 - accuracy: 0.7679 - val_loss: 1.3507 - val_accuracy: 0.7025 Epoch 99/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8792 - accuracy: 0.7690 - val_loss: 1.3510 - val_accuracy: 0.7046 Epoch 100/100 1148/1148 [==============================] - 5s 5ms/step - loss: 0.8803 - accuracy: 0.7680 - val_loss: 1.3526 - val_accuracy: 0.7006
GRU_V1.summary()
Model: "gru_v1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 128) 53760
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 220,421
Trainable params: 220,421
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_V1_history.history)
Observations
GRU_V1.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.3419 - accuracy: 0.6987
[1.3419456481933594, 0.6986760497093201]
Observations
tf.keras.backend.clear_session()
# Create the model
GRU_V2 = Sequential(
name='gru_v2',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(256, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_V2.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_V2_history = GRU_V2.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 7s 5ms/step - loss: 4.7888 - accuracy: 0.1552 - val_loss: 3.8862 - val_accuracy: 0.2642 Epoch 2/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.3270 - accuracy: 0.3304 - val_loss: 2.9117 - val_accuracy: 0.3966 Epoch 3/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5623 - accuracy: 0.4435 - val_loss: 2.3888 - val_accuracy: 0.4919 Epoch 4/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1084 - accuracy: 0.5271 - val_loss: 2.0847 - val_accuracy: 0.5474 Epoch 5/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.8298 - accuracy: 0.5756 - val_loss: 1.8946 - val_accuracy: 0.5863 Epoch 6/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.6393 - accuracy: 0.6160 - val_loss: 1.7773 - val_accuracy: 0.6041 Epoch 7/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.5140 - accuracy: 0.6396 - val_loss: 1.6840 - val_accuracy: 0.6254 Epoch 8/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.4087 - accuracy: 0.6623 - val_loss: 1.6416 - val_accuracy: 0.6331 Epoch 9/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3300 - accuracy: 0.6781 - val_loss: 1.5811 - val_accuracy: 0.6482 Epoch 10/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.2717 - accuracy: 0.6912 - val_loss: 1.5566 - val_accuracy: 0.6534 Epoch 11/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2281 - accuracy: 0.7008 - val_loss: 1.5285 - val_accuracy: 0.6604 Epoch 12/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1784 - accuracy: 0.7120 - val_loss: 1.5198 - val_accuracy: 0.6603 Epoch 13/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1433 - accuracy: 0.7172 - val_loss: 1.4867 - val_accuracy: 0.6715 Epoch 14/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1084 - accuracy: 0.7256 - val_loss: 1.4887 - val_accuracy: 0.6644 Epoch 15/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0897 - accuracy: 0.7290 - val_loss: 1.4667 - val_accuracy: 0.6725 Epoch 16/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0612 - accuracy: 0.7338 - val_loss: 1.4685 - val_accuracy: 0.6735 Epoch 17/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0357 - accuracy: 0.7395 - val_loss: 1.4607 - val_accuracy: 0.6755 Epoch 18/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0225 - accuracy: 0.7419 - val_loss: 1.4395 - val_accuracy: 0.6764 Epoch 19/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0085 - accuracy: 0.7433 - val_loss: 1.4322 - val_accuracy: 0.6816 Epoch 20/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9919 - accuracy: 0.7504 - val_loss: 1.4385 - val_accuracy: 0.6817 Epoch 21/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9806 - accuracy: 0.7477 - val_loss: 1.4360 - val_accuracy: 0.6835 Epoch 22/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9623 - accuracy: 0.7528 - val_loss: 1.4338 - val_accuracy: 0.6854 Epoch 23/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9542 - accuracy: 0.7546 - val_loss: 1.4346 - val_accuracy: 0.6848 Epoch 24/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9411 - accuracy: 0.7584 - val_loss: 1.4191 - val_accuracy: 0.6865 Epoch 25/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9369 - accuracy: 0.7600 - val_loss: 1.4087 - val_accuracy: 0.6876 Epoch 26/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9298 - accuracy: 0.7621 - val_loss: 1.4268 - val_accuracy: 0.6867 Epoch 27/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9194 - accuracy: 0.7630 - val_loss: 1.4332 - val_accuracy: 0.6879 Epoch 28/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9132 - accuracy: 0.7641 - val_loss: 1.4306 - val_accuracy: 0.6871 Epoch 29/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9115 - accuracy: 0.7631 - val_loss: 1.4191 - val_accuracy: 0.6907 Epoch 30/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8967 - accuracy: 0.7676 - val_loss: 1.4246 - val_accuracy: 0.6918 Epoch 31/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8903 - accuracy: 0.7672 - val_loss: 1.4303 - val_accuracy: 0.6914 Epoch 32/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8916 - accuracy: 0.7685 - val_loss: 1.4313 - val_accuracy: 0.6905 Epoch 33/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8860 - accuracy: 0.7684 - val_loss: 1.4170 - val_accuracy: 0.6917 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8815 - accuracy: 0.7688 - val_loss: 1.4229 - val_accuracy: 0.6949 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8797 - accuracy: 0.7696 - val_loss: 1.4172 - val_accuracy: 0.6969 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8738 - accuracy: 0.7718 - val_loss: 1.4183 - val_accuracy: 0.6943 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8660 - accuracy: 0.7724 - val_loss: 1.4151 - val_accuracy: 0.6966 Epoch 38/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8694 - accuracy: 0.7728 - val_loss: 1.4329 - val_accuracy: 0.6941 Epoch 39/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8628 - accuracy: 0.7737 - val_loss: 1.4294 - val_accuracy: 0.6959 Epoch 40/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8639 - accuracy: 0.7722 - val_loss: 1.4209 - val_accuracy: 0.6939 Epoch 41/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8500 - accuracy: 0.7770 - val_loss: 1.4169 - val_accuracy: 0.6916 Epoch 42/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8514 - accuracy: 0.7766 - val_loss: 1.4207 - val_accuracy: 0.6961 Epoch 43/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8480 - accuracy: 0.7767 - val_loss: 1.4236 - val_accuracy: 0.6968 Epoch 44/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8481 - accuracy: 0.7752 - val_loss: 1.4336 - val_accuracy: 0.6958 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8432 - accuracy: 0.7755 - val_loss: 1.4340 - val_accuracy: 0.6992 Epoch 46/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8437 - accuracy: 0.7785 - val_loss: 1.4275 - val_accuracy: 0.6955 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8449 - accuracy: 0.7757 - val_loss: 1.4341 - val_accuracy: 0.6931 Epoch 48/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8397 - accuracy: 0.7779 - val_loss: 1.4198 - val_accuracy: 0.6994 Epoch 49/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8397 - accuracy: 0.7783 - val_loss: 1.4175 - val_accuracy: 0.6961 Epoch 50/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8353 - accuracy: 0.7784 - val_loss: 1.4177 - val_accuracy: 0.6959 Epoch 51/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8380 - accuracy: 0.7792 - val_loss: 1.4246 - val_accuracy: 0.6997 Epoch 52/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8343 - accuracy: 0.7778 - val_loss: 1.4358 - val_accuracy: 0.6970 Epoch 53/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8311 - accuracy: 0.7788 - val_loss: 1.4301 - val_accuracy: 0.6937 Epoch 54/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8280 - accuracy: 0.7789 - val_loss: 1.4268 - val_accuracy: 0.6966 Epoch 55/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8226 - accuracy: 0.7797 - val_loss: 1.4251 - val_accuracy: 0.7000 Epoch 56/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8292 - accuracy: 0.7791 - val_loss: 1.4213 - val_accuracy: 0.7000 Epoch 57/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8213 - accuracy: 0.7815 - val_loss: 1.4326 - val_accuracy: 0.6981 Epoch 58/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8261 - accuracy: 0.7789 - val_loss: 1.4215 - val_accuracy: 0.6992 Epoch 59/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8181 - accuracy: 0.7800 - val_loss: 1.4313 - val_accuracy: 0.7003 Epoch 60/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8218 - accuracy: 0.7797 - val_loss: 1.4200 - val_accuracy: 0.7001 Epoch 61/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8157 - accuracy: 0.7813 - val_loss: 1.4166 - val_accuracy: 0.6970 Epoch 62/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8156 - accuracy: 0.7800 - val_loss: 1.4383 - val_accuracy: 0.6952 Epoch 63/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8068 - accuracy: 0.7823 - val_loss: 1.4336 - val_accuracy: 0.6997 Epoch 64/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8171 - accuracy: 0.7807 - val_loss: 1.4376 - val_accuracy: 0.6979 Epoch 65/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8103 - accuracy: 0.7821 - val_loss: 1.4448 - val_accuracy: 0.6987 Epoch 66/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8144 - accuracy: 0.7801 - val_loss: 1.4362 - val_accuracy: 0.7006 Epoch 67/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8119 - accuracy: 0.7822 - val_loss: 1.4249 - val_accuracy: 0.6996 Epoch 68/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8153 - accuracy: 0.7805 - val_loss: 1.4284 - val_accuracy: 0.6992 Epoch 69/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8047 - accuracy: 0.7825 - val_loss: 1.4247 - val_accuracy: 0.6999 Epoch 70/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8144 - accuracy: 0.7808 - val_loss: 1.4390 - val_accuracy: 0.6999 Epoch 71/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8092 - accuracy: 0.7816 - val_loss: 1.4351 - val_accuracy: 0.6973 Epoch 72/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8117 - accuracy: 0.7804 - val_loss: 1.4325 - val_accuracy: 0.6977 Epoch 73/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8049 - accuracy: 0.7829 - val_loss: 1.4335 - val_accuracy: 0.6998 Epoch 74/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8026 - accuracy: 0.7837 - val_loss: 1.4234 - val_accuracy: 0.6997 Epoch 75/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8016 - accuracy: 0.7831 - val_loss: 1.4563 - val_accuracy: 0.6968 Epoch 76/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8058 - accuracy: 0.7822 - val_loss: 1.4460 - val_accuracy: 0.6969 Epoch 77/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8023 - accuracy: 0.7820 - val_loss: 1.4475 - val_accuracy: 0.6983 Epoch 78/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8000 - accuracy: 0.7829 - val_loss: 1.4299 - val_accuracy: 0.7011 Epoch 79/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8018 - accuracy: 0.7811 - val_loss: 1.4388 - val_accuracy: 0.7014 Epoch 80/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7993 - accuracy: 0.7828 - val_loss: 1.4354 - val_accuracy: 0.6995 Epoch 81/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7964 - accuracy: 0.7827 - val_loss: 1.4476 - val_accuracy: 0.6988 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7986 - accuracy: 0.7827 - val_loss: 1.4404 - val_accuracy: 0.6986 Epoch 83/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7980 - accuracy: 0.7837 - val_loss: 1.4322 - val_accuracy: 0.7024 Epoch 84/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.8009 - accuracy: 0.7818 - val_loss: 1.4443 - val_accuracy: 0.6986 Epoch 85/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7989 - accuracy: 0.7823 - val_loss: 1.4467 - val_accuracy: 0.6997 Epoch 86/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7984 - accuracy: 0.7830 - val_loss: 1.4336 - val_accuracy: 0.7024 Epoch 87/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7903 - accuracy: 0.7841 - val_loss: 1.4425 - val_accuracy: 0.7015 Epoch 88/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7942 - accuracy: 0.7840 - val_loss: 1.4277 - val_accuracy: 0.7041 Epoch 89/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7910 - accuracy: 0.7832 - val_loss: 1.4169 - val_accuracy: 0.7042 Epoch 90/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7865 - accuracy: 0.7863 - val_loss: 1.4456 - val_accuracy: 0.7021 Epoch 91/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7937 - accuracy: 0.7836 - val_loss: 1.4345 - val_accuracy: 0.7007 Epoch 92/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7924 - accuracy: 0.7847 - val_loss: 1.4281 - val_accuracy: 0.7016 Epoch 93/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7936 - accuracy: 0.7846 - val_loss: 1.4264 - val_accuracy: 0.7010 Epoch 94/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7939 - accuracy: 0.7833 - val_loss: 1.4340 - val_accuracy: 0.7036 Epoch 95/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7909 - accuracy: 0.7836 - val_loss: 1.4232 - val_accuracy: 0.7009 Epoch 96/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7922 - accuracy: 0.7826 - val_loss: 1.4381 - val_accuracy: 0.7007 Epoch 97/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7870 - accuracy: 0.7840 - val_loss: 1.4282 - val_accuracy: 0.7030 Epoch 98/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7850 - accuracy: 0.7855 - val_loss: 1.4431 - val_accuracy: 0.7012 Epoch 99/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7898 - accuracy: 0.7857 - val_loss: 1.4473 - val_accuracy: 0.7002 Epoch 100/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7894 - accuracy: 0.7836 - val_loss: 1.4279 - val_accuracy: 0.7045
GRU_V2.summary()
Model: "gru_v2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 256) 205824
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 525,957
Trainable params: 525,957
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_V2_history.history)
Observations
GRU_V2.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.4258 - accuracy: 0.7071
[1.4258129596710205, 0.7070938348770142]
Observations
tf.keras.backend.clear_session()
# Create the model
GRU_V3 = Sequential(
name='gru_v3',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(64, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_V3.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_V3_history = GRU_V3.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 6s 5ms/step - loss: 5.1391 - accuracy: 0.1156 - val_loss: 4.5869 - val_accuracy: 0.1491 Epoch 2/100 1148/1148 [==============================] - 5s 5ms/step - loss: 4.2481 - accuracy: 0.2000 - val_loss: 3.9412 - val_accuracy: 0.2390 Epoch 3/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.6982 - accuracy: 0.2702 - val_loss: 3.4665 - val_accuracy: 0.3133 Epoch 4/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.3008 - accuracy: 0.3203 - val_loss: 3.1474 - val_accuracy: 0.3588 Epoch 5/100 1148/1148 [==============================] - 5s 5ms/step - loss: 3.0204 - accuracy: 0.3608 - val_loss: 2.9128 - val_accuracy: 0.3931 Epoch 6/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.8109 - accuracy: 0.3927 - val_loss: 2.7280 - val_accuracy: 0.4205 Epoch 7/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.6393 - accuracy: 0.4179 - val_loss: 2.5827 - val_accuracy: 0.4514 Epoch 8/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.5056 - accuracy: 0.4440 - val_loss: 2.4583 - val_accuracy: 0.4692 Epoch 9/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.3837 - accuracy: 0.4636 - val_loss: 2.3498 - val_accuracy: 0.4914 Epoch 10/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.2846 - accuracy: 0.4811 - val_loss: 2.2567 - val_accuracy: 0.5103 Epoch 11/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.2016 - accuracy: 0.4982 - val_loss: 2.1743 - val_accuracy: 0.5235 Epoch 12/100 1148/1148 [==============================] - 5s 4ms/step - loss: 2.1296 - accuracy: 0.5106 - val_loss: 2.1009 - val_accuracy: 0.5412 Epoch 13/100 1148/1148 [==============================] - 5s 5ms/step - loss: 2.0580 - accuracy: 0.5240 - val_loss: 2.0522 - val_accuracy: 0.5459 Epoch 14/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0008 - accuracy: 0.5336 - val_loss: 2.0006 - val_accuracy: 0.5562 Epoch 15/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9437 - accuracy: 0.5473 - val_loss: 1.9496 - val_accuracy: 0.5673 Epoch 16/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9027 - accuracy: 0.5522 - val_loss: 1.9070 - val_accuracy: 0.5746 Epoch 17/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.8639 - accuracy: 0.5616 - val_loss: 1.8756 - val_accuracy: 0.5812 Epoch 18/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.8200 - accuracy: 0.5686 - val_loss: 1.8269 - val_accuracy: 0.5928 Epoch 19/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7851 - accuracy: 0.5776 - val_loss: 1.8137 - val_accuracy: 0.5859 Epoch 20/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7488 - accuracy: 0.5830 - val_loss: 1.7810 - val_accuracy: 0.6015 Epoch 21/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.7237 - accuracy: 0.5893 - val_loss: 1.7571 - val_accuracy: 0.6041 Epoch 22/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6927 - accuracy: 0.5974 - val_loss: 1.7432 - val_accuracy: 0.6086 Epoch 23/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6684 - accuracy: 0.5974 - val_loss: 1.7174 - val_accuracy: 0.6151 Epoch 24/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6473 - accuracy: 0.6062 - val_loss: 1.6955 - val_accuracy: 0.6183 Epoch 25/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6312 - accuracy: 0.6108 - val_loss: 1.6722 - val_accuracy: 0.6245 Epoch 26/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.6030 - accuracy: 0.6127 - val_loss: 1.6571 - val_accuracy: 0.6249 Epoch 27/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5877 - accuracy: 0.6162 - val_loss: 1.6482 - val_accuracy: 0.6267 Epoch 28/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5643 - accuracy: 0.6235 - val_loss: 1.6282 - val_accuracy: 0.6310 Epoch 29/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5613 - accuracy: 0.6223 - val_loss: 1.6155 - val_accuracy: 0.6322 Epoch 30/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.5317 - accuracy: 0.6308 - val_loss: 1.6089 - val_accuracy: 0.6357 Epoch 31/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.5199 - accuracy: 0.6303 - val_loss: 1.5919 - val_accuracy: 0.6365 Epoch 32/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.5100 - accuracy: 0.6319 - val_loss: 1.5769 - val_accuracy: 0.6415 Epoch 33/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.4935 - accuracy: 0.6375 - val_loss: 1.5643 - val_accuracy: 0.6415 Epoch 34/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4854 - accuracy: 0.6371 - val_loss: 1.5520 - val_accuracy: 0.6483 Epoch 35/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4770 - accuracy: 0.6413 - val_loss: 1.5435 - val_accuracy: 0.6474 Epoch 36/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4653 - accuracy: 0.6425 - val_loss: 1.5395 - val_accuracy: 0.6465 Epoch 37/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.4481 - accuracy: 0.6459 - val_loss: 1.5379 - val_accuracy: 0.6483 Epoch 38/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.4341 - accuracy: 0.6499 - val_loss: 1.5339 - val_accuracy: 0.6486 Epoch 39/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.4278 - accuracy: 0.6515 - val_loss: 1.5211 - val_accuracy: 0.6525 Epoch 40/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4260 - accuracy: 0.6493 - val_loss: 1.5155 - val_accuracy: 0.6529 Epoch 41/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4106 - accuracy: 0.6518 - val_loss: 1.5126 - val_accuracy: 0.6537 Epoch 42/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.4040 - accuracy: 0.6552 - val_loss: 1.5028 - val_accuracy: 0.6563 Epoch 43/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3964 - accuracy: 0.6542 - val_loss: 1.5080 - val_accuracy: 0.6550 Epoch 44/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3863 - accuracy: 0.6593 - val_loss: 1.5012 - val_accuracy: 0.6568 Epoch 45/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3742 - accuracy: 0.6619 - val_loss: 1.4869 - val_accuracy: 0.6608 Epoch 46/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3730 - accuracy: 0.6596 - val_loss: 1.4891 - val_accuracy: 0.6602 Epoch 47/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3701 - accuracy: 0.6597 - val_loss: 1.4792 - val_accuracy: 0.6606 Epoch 48/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3649 - accuracy: 0.6611 - val_loss: 1.4741 - val_accuracy: 0.6647 Epoch 49/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3464 - accuracy: 0.6670 - val_loss: 1.4727 - val_accuracy: 0.6591 Epoch 50/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3519 - accuracy: 0.6643 - val_loss: 1.4668 - val_accuracy: 0.6657 Epoch 51/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3470 - accuracy: 0.6672 - val_loss: 1.4647 - val_accuracy: 0.6612 Epoch 52/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3381 - accuracy: 0.6705 - val_loss: 1.4584 - val_accuracy: 0.6665 Epoch 53/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3294 - accuracy: 0.6732 - val_loss: 1.4611 - val_accuracy: 0.6665 Epoch 54/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3222 - accuracy: 0.6722 - val_loss: 1.4580 - val_accuracy: 0.6668 Epoch 55/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3247 - accuracy: 0.6737 - val_loss: 1.4443 - val_accuracy: 0.6670 Epoch 56/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.3197 - accuracy: 0.6736 - val_loss: 1.4502 - val_accuracy: 0.6663 Epoch 57/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3154 - accuracy: 0.6747 - val_loss: 1.4387 - val_accuracy: 0.6701 Epoch 58/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3037 - accuracy: 0.6761 - val_loss: 1.4374 - val_accuracy: 0.6682 Epoch 59/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.3041 - accuracy: 0.6764 - val_loss: 1.4455 - val_accuracy: 0.6724 Epoch 60/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2965 - accuracy: 0.6782 - val_loss: 1.4418 - val_accuracy: 0.6703 Epoch 61/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2846 - accuracy: 0.6774 - val_loss: 1.4331 - val_accuracy: 0.6708 Epoch 62/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2923 - accuracy: 0.6785 - val_loss: 1.4378 - val_accuracy: 0.6686 Epoch 63/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2848 - accuracy: 0.6820 - val_loss: 1.4344 - val_accuracy: 0.6708 Epoch 64/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2809 - accuracy: 0.6837 - val_loss: 1.4303 - val_accuracy: 0.6700 Epoch 65/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2763 - accuracy: 0.6816 - val_loss: 1.4400 - val_accuracy: 0.6724 Epoch 66/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2744 - accuracy: 0.6849 - val_loss: 1.4277 - val_accuracy: 0.6724 Epoch 67/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2662 - accuracy: 0.6823 - val_loss: 1.4210 - val_accuracy: 0.6701 Epoch 68/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2570 - accuracy: 0.6870 - val_loss: 1.4243 - val_accuracy: 0.6720 Epoch 69/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2558 - accuracy: 0.6883 - val_loss: 1.4210 - val_accuracy: 0.6719 Epoch 70/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2555 - accuracy: 0.6881 - val_loss: 1.4186 - val_accuracy: 0.6755 Epoch 71/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2563 - accuracy: 0.6856 - val_loss: 1.4175 - val_accuracy: 0.6722 Epoch 72/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2448 - accuracy: 0.6890 - val_loss: 1.4164 - val_accuracy: 0.6740 Epoch 73/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2477 - accuracy: 0.6888 - val_loss: 1.4053 - val_accuracy: 0.6773 Epoch 74/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2411 - accuracy: 0.6898 - val_loss: 1.4145 - val_accuracy: 0.6740 Epoch 75/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2456 - accuracy: 0.6904 - val_loss: 1.4068 - val_accuracy: 0.6765 Epoch 76/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2404 - accuracy: 0.6902 - val_loss: 1.4121 - val_accuracy: 0.6723 Epoch 77/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2446 - accuracy: 0.6882 - val_loss: 1.4029 - val_accuracy: 0.6774 Epoch 78/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2390 - accuracy: 0.6907 - val_loss: 1.4075 - val_accuracy: 0.6781 Epoch 79/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2324 - accuracy: 0.6903 - val_loss: 1.4042 - val_accuracy: 0.6766 Epoch 80/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2285 - accuracy: 0.6903 - val_loss: 1.4027 - val_accuracy: 0.6775 Epoch 81/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.2190 - accuracy: 0.6933 - val_loss: 1.3927 - val_accuracy: 0.6784 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2138 - accuracy: 0.6947 - val_loss: 1.4019 - val_accuracy: 0.6751 Epoch 83/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2267 - accuracy: 0.6928 - val_loss: 1.3961 - val_accuracy: 0.6796 Epoch 84/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2112 - accuracy: 0.6961 - val_loss: 1.4015 - val_accuracy: 0.6782 Epoch 85/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2156 - accuracy: 0.6953 - val_loss: 1.3992 - val_accuracy: 0.6793 Epoch 86/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2074 - accuracy: 0.6979 - val_loss: 1.3963 - val_accuracy: 0.6804 Epoch 87/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2192 - accuracy: 0.6920 - val_loss: 1.3926 - val_accuracy: 0.6795 Epoch 88/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1997 - accuracy: 0.6996 - val_loss: 1.3998 - val_accuracy: 0.6794 Epoch 89/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2084 - accuracy: 0.6987 - val_loss: 1.3987 - val_accuracy: 0.6784 Epoch 90/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2001 - accuracy: 0.7001 - val_loss: 1.4007 - val_accuracy: 0.6802 Epoch 91/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1985 - accuracy: 0.7003 - val_loss: 1.3944 - val_accuracy: 0.6787 Epoch 92/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1922 - accuracy: 0.7001 - val_loss: 1.3857 - val_accuracy: 0.6803 Epoch 93/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.2027 - accuracy: 0.6963 - val_loss: 1.3874 - val_accuracy: 0.6865 Epoch 94/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1937 - accuracy: 0.7004 - val_loss: 1.3859 - val_accuracy: 0.6835 Epoch 95/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1836 - accuracy: 0.7007 - val_loss: 1.3853 - val_accuracy: 0.6831 Epoch 96/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1884 - accuracy: 0.7004 - val_loss: 1.3921 - val_accuracy: 0.6787 Epoch 97/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1929 - accuracy: 0.6998 - val_loss: 1.3915 - val_accuracy: 0.6802 Epoch 98/100 1148/1148 [==============================] - 5s 5ms/step - loss: 1.1804 - accuracy: 0.7045 - val_loss: 1.3864 - val_accuracy: 0.6831 Epoch 99/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.1840 - accuracy: 0.7032 - val_loss: 1.3773 - val_accuracy: 0.6822 Epoch 100/100 1148/1148 [==============================] - 5s 4ms/step - loss: 1.1729 - accuracy: 0.7044 - val_loss: 1.3812 - val_accuracy: 0.6849
GRU_V3.summary()
Model: "gru_v3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 64) 14592
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 104,517
Trainable params: 104,517
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_V3_history.history)
Observations
GRU_V3.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.3736 - accuracy: 0.6818
[1.373557686805725, 0.6818404793739319]
Observations
tf.keras.backend.clear_session()
# Create the model
GRU_V4 = Sequential(
name='gru_v4',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(256, activation='tanh', return_sequences=True),
GRU(128, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_V4.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_V4_history = GRU_V4.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 10s 8ms/step - loss: 5.4663 - accuracy: 0.0835 - val_loss: 5.0312 - val_accuracy: 0.1268 Epoch 2/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.8376 - accuracy: 0.1427 - val_loss: 4.6088 - val_accuracy: 0.1737 Epoch 3/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.5230 - accuracy: 0.1801 - val_loss: 4.3765 - val_accuracy: 0.2043 Epoch 4/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.3408 - accuracy: 0.1986 - val_loss: 4.2243 - val_accuracy: 0.2141 Epoch 5/100 1148/1148 [==============================] - 9s 7ms/step - loss: 4.2112 - accuracy: 0.2110 - val_loss: 4.0837 - val_accuracy: 0.2321 Epoch 6/100 1148/1148 [==============================] - 9s 8ms/step - loss: 4.0949 - accuracy: 0.2243 - val_loss: 3.9840 - val_accuracy: 0.2451 Epoch 7/100 1148/1148 [==============================] - 9s 7ms/step - loss: 3.9971 - accuracy: 0.2337 - val_loss: 3.8648 - val_accuracy: 0.2518 Epoch 8/100 1148/1148 [==============================] - 9s 7ms/step - loss: 3.8843 - accuracy: 0.2446 - val_loss: 3.7575 - val_accuracy: 0.2684 Epoch 9/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.7750 - accuracy: 0.2577 - val_loss: 3.6405 - val_accuracy: 0.2918 Epoch 10/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.6563 - accuracy: 0.2715 - val_loss: 3.5188 - val_accuracy: 0.3115 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.5360 - accuracy: 0.2882 - val_loss: 3.3890 - val_accuracy: 0.3213 Epoch 12/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.4201 - accuracy: 0.3030 - val_loss: 3.2634 - val_accuracy: 0.3437 Epoch 13/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.3099 - accuracy: 0.3185 - val_loss: 3.1497 - val_accuracy: 0.3607 Epoch 14/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.2018 - accuracy: 0.3326 - val_loss: 3.0478 - val_accuracy: 0.3716 Epoch 15/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.0936 - accuracy: 0.3481 - val_loss: 2.9531 - val_accuracy: 0.3882 Epoch 16/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.9817 - accuracy: 0.3667 - val_loss: 2.8503 - val_accuracy: 0.4004 Epoch 17/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.8966 - accuracy: 0.3780 - val_loss: 2.7622 - val_accuracy: 0.4184 Epoch 18/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.7927 - accuracy: 0.3911 - val_loss: 2.6606 - val_accuracy: 0.4323 Epoch 19/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.7018 - accuracy: 0.4076 - val_loss: 2.5456 - val_accuracy: 0.4525 Epoch 20/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.5970 - accuracy: 0.4268 - val_loss: 2.4695 - val_accuracy: 0.4619 Epoch 21/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.4879 - accuracy: 0.4426 - val_loss: 2.3742 - val_accuracy: 0.4805 Epoch 22/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3925 - accuracy: 0.4592 - val_loss: 2.2793 - val_accuracy: 0.4946 Epoch 23/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3009 - accuracy: 0.4767 - val_loss: 2.1936 - val_accuracy: 0.5152 Epoch 24/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.2173 - accuracy: 0.4920 - val_loss: 2.1246 - val_accuracy: 0.5271 Epoch 25/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1349 - accuracy: 0.5063 - val_loss: 2.0446 - val_accuracy: 0.5458 Epoch 26/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.0521 - accuracy: 0.5218 - val_loss: 1.9776 - val_accuracy: 0.5562 Epoch 27/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9791 - accuracy: 0.5356 - val_loss: 1.9313 - val_accuracy: 0.5656 Epoch 28/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9143 - accuracy: 0.5483 - val_loss: 1.8855 - val_accuracy: 0.5771 Epoch 29/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8631 - accuracy: 0.5599 - val_loss: 1.8245 - val_accuracy: 0.5886 Epoch 30/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8069 - accuracy: 0.5703 - val_loss: 1.7899 - val_accuracy: 0.5968 Epoch 31/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.7619 - accuracy: 0.5786 - val_loss: 1.7562 - val_accuracy: 0.6009 Epoch 32/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.7105 - accuracy: 0.5898 - val_loss: 1.7420 - val_accuracy: 0.6059 Epoch 33/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.6698 - accuracy: 0.5980 - val_loss: 1.6835 - val_accuracy: 0.6208 Epoch 34/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.6389 - accuracy: 0.6043 - val_loss: 1.6626 - val_accuracy: 0.6205 Epoch 35/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6050 - accuracy: 0.6103 - val_loss: 1.6315 - val_accuracy: 0.6299 Epoch 36/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5646 - accuracy: 0.6203 - val_loss: 1.6174 - val_accuracy: 0.6338 Epoch 37/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.5395 - accuracy: 0.6246 - val_loss: 1.5979 - val_accuracy: 0.6388 Epoch 38/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5074 - accuracy: 0.6326 - val_loss: 1.5748 - val_accuracy: 0.6401 Epoch 39/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4825 - accuracy: 0.6399 - val_loss: 1.5638 - val_accuracy: 0.6429 Epoch 40/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4619 - accuracy: 0.6445 - val_loss: 1.5412 - val_accuracy: 0.6486 Epoch 41/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4458 - accuracy: 0.6470 - val_loss: 1.5412 - val_accuracy: 0.6500 Epoch 42/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4256 - accuracy: 0.6492 - val_loss: 1.5271 - val_accuracy: 0.6477 Epoch 43/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3974 - accuracy: 0.6555 - val_loss: 1.5038 - val_accuracy: 0.6570 Epoch 44/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3814 - accuracy: 0.6599 - val_loss: 1.4877 - val_accuracy: 0.6603 Epoch 45/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3618 - accuracy: 0.6628 - val_loss: 1.4960 - val_accuracy: 0.6635 Epoch 46/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3393 - accuracy: 0.6678 - val_loss: 1.4712 - val_accuracy: 0.6661 Epoch 47/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3307 - accuracy: 0.6689 - val_loss: 1.4656 - val_accuracy: 0.6686 Epoch 48/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3198 - accuracy: 0.6726 - val_loss: 1.4560 - val_accuracy: 0.6666 Epoch 49/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.2987 - accuracy: 0.6777 - val_loss: 1.4577 - val_accuracy: 0.6726 Epoch 50/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2818 - accuracy: 0.6807 - val_loss: 1.4503 - val_accuracy: 0.6715 Epoch 51/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.2709 - accuracy: 0.6838 - val_loss: 1.4430 - val_accuracy: 0.6693 Epoch 52/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.2556 - accuracy: 0.6906 - val_loss: 1.4311 - val_accuracy: 0.6732 Epoch 53/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2478 - accuracy: 0.6902 - val_loss: 1.4281 - val_accuracy: 0.6742 Epoch 54/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2387 - accuracy: 0.6940 - val_loss: 1.4224 - val_accuracy: 0.6762 Epoch 55/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2269 - accuracy: 0.6931 - val_loss: 1.4114 - val_accuracy: 0.6811 Epoch 56/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2156 - accuracy: 0.6965 - val_loss: 1.4201 - val_accuracy: 0.6755 Epoch 57/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2041 - accuracy: 0.6967 - val_loss: 1.4038 - val_accuracy: 0.6820 Epoch 58/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1955 - accuracy: 0.7018 - val_loss: 1.4079 - val_accuracy: 0.6799 Epoch 59/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1899 - accuracy: 0.7035 - val_loss: 1.4004 - val_accuracy: 0.6813 Epoch 60/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1794 - accuracy: 0.7052 - val_loss: 1.4012 - val_accuracy: 0.6837 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1698 - accuracy: 0.7088 - val_loss: 1.3956 - val_accuracy: 0.6881 Epoch 62/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.1586 - accuracy: 0.7116 - val_loss: 1.3875 - val_accuracy: 0.6836 Epoch 63/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.1506 - accuracy: 0.7126 - val_loss: 1.3794 - val_accuracy: 0.6857 Epoch 64/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1475 - accuracy: 0.7105 - val_loss: 1.3853 - val_accuracy: 0.6847 Epoch 65/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1320 - accuracy: 0.7146 - val_loss: 1.3736 - val_accuracy: 0.6859 Epoch 66/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1284 - accuracy: 0.7180 - val_loss: 1.3803 - val_accuracy: 0.6863 Epoch 67/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1273 - accuracy: 0.7160 - val_loss: 1.3858 - val_accuracy: 0.6853 Epoch 68/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1133 - accuracy: 0.7233 - val_loss: 1.3765 - val_accuracy: 0.6878 Epoch 69/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1049 - accuracy: 0.7195 - val_loss: 1.3647 - val_accuracy: 0.6874 Epoch 70/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1056 - accuracy: 0.7226 - val_loss: 1.3718 - val_accuracy: 0.6886 Epoch 71/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0958 - accuracy: 0.7235 - val_loss: 1.3652 - val_accuracy: 0.6924 Epoch 72/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0945 - accuracy: 0.7241 - val_loss: 1.3661 - val_accuracy: 0.6934 Epoch 73/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0839 - accuracy: 0.7246 - val_loss: 1.3733 - val_accuracy: 0.6872 Epoch 74/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0813 - accuracy: 0.7272 - val_loss: 1.3651 - val_accuracy: 0.6894 Epoch 75/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0761 - accuracy: 0.7269 - val_loss: 1.3594 - val_accuracy: 0.6928 Epoch 76/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0742 - accuracy: 0.7282 - val_loss: 1.3583 - val_accuracy: 0.6918 Epoch 77/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0692 - accuracy: 0.7288 - val_loss: 1.3496 - val_accuracy: 0.6942 Epoch 78/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0648 - accuracy: 0.7303 - val_loss: 1.3665 - val_accuracy: 0.6945 Epoch 79/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0586 - accuracy: 0.7342 - val_loss: 1.3597 - val_accuracy: 0.6921 Epoch 80/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0497 - accuracy: 0.7334 - val_loss: 1.3505 - val_accuracy: 0.6943 Epoch 81/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0414 - accuracy: 0.7343 - val_loss: 1.3455 - val_accuracy: 0.6966 Epoch 82/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0393 - accuracy: 0.7368 - val_loss: 1.3606 - val_accuracy: 0.6970 Epoch 83/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0406 - accuracy: 0.7356 - val_loss: 1.3523 - val_accuracy: 0.6944 Epoch 84/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0311 - accuracy: 0.7389 - val_loss: 1.3455 - val_accuracy: 0.6995 Epoch 85/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0227 - accuracy: 0.7391 - val_loss: 1.3458 - val_accuracy: 0.6979 Epoch 86/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0279 - accuracy: 0.7403 - val_loss: 1.3522 - val_accuracy: 0.6973 Epoch 87/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0239 - accuracy: 0.7394 - val_loss: 1.3408 - val_accuracy: 0.7004 Epoch 88/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0160 - accuracy: 0.7417 - val_loss: 1.3519 - val_accuracy: 0.6992 Epoch 89/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0217 - accuracy: 0.7414 - val_loss: 1.3505 - val_accuracy: 0.6970 Epoch 90/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0099 - accuracy: 0.7418 - val_loss: 1.3422 - val_accuracy: 0.7025 Epoch 91/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0075 - accuracy: 0.7420 - val_loss: 1.3425 - val_accuracy: 0.7013 Epoch 92/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0038 - accuracy: 0.7434 - val_loss: 1.3429 - val_accuracy: 0.6980 Epoch 93/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0017 - accuracy: 0.7455 - val_loss: 1.3490 - val_accuracy: 0.6984 Epoch 94/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0054 - accuracy: 0.7446 - val_loss: 1.3560 - val_accuracy: 0.7012 Epoch 95/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9970 - accuracy: 0.7449 - val_loss: 1.3330 - val_accuracy: 0.7015 Epoch 96/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9870 - accuracy: 0.7495 - val_loss: 1.3319 - val_accuracy: 0.7063 Epoch 97/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9902 - accuracy: 0.7485 - val_loss: 1.3445 - val_accuracy: 0.6990 Epoch 98/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9930 - accuracy: 0.7450 - val_loss: 1.3424 - val_accuracy: 0.6997 Epoch 99/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9816 - accuracy: 0.7502 - val_loss: 1.3371 - val_accuracy: 0.7020 Epoch 100/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9824 - accuracy: 0.7491 - val_loss: 1.3449 - val_accuracy: 0.7024
GRU_V4.summary()
Model: "gru_v4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 34, 256) 205824
gru_1 (GRU) (None, 128) 148224
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 520,709
Trainable params: 520,709
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_V4_history.history)
Observations
GRU_V4.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.3385 - accuracy: 0.7028
[1.3385025262832642, 0.7027623653411865]
Observations
![]()
Image Source: Kumari, K., 2023
# Bidirectional LSTM
tf.keras.backend.clear_session()
# Create the model
Bi_LSTM_V1 = Sequential(
name='bi_directional_lstm_v1',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(LSTM(256, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_LSTM_V1.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_LSTM_V1_history = Bi_LSTM_V1.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 11s 8ms/step - loss: 4.9760 - accuracy: 0.1329 - val_loss: 4.1694 - val_accuracy: 0.2207 Epoch 2/100 1148/1148 [==============================] - 9s 8ms/step - loss: 3.6301 - accuracy: 0.2848 - val_loss: 3.2637 - val_accuracy: 0.3457 Epoch 3/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.9039 - accuracy: 0.3908 - val_loss: 2.7701 - val_accuracy: 0.4268 Epoch 4/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.4648 - accuracy: 0.4613 - val_loss: 2.4488 - val_accuracy: 0.4800 Epoch 5/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.1759 - accuracy: 0.5065 - val_loss: 2.2286 - val_accuracy: 0.5151 Epoch 6/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.9571 - accuracy: 0.5495 - val_loss: 2.0640 - val_accuracy: 0.5436 Epoch 7/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.7940 - accuracy: 0.5775 - val_loss: 1.9345 - val_accuracy: 0.5698 Epoch 8/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.6597 - accuracy: 0.6043 - val_loss: 1.8374 - val_accuracy: 0.5888 Epoch 9/100 1148/1148 [==============================] - 10s 9ms/step - loss: 1.5580 - accuracy: 0.6247 - val_loss: 1.7583 - val_accuracy: 0.6065 Epoch 10/100 1148/1148 [==============================] - 10s 9ms/step - loss: 1.4613 - accuracy: 0.6465 - val_loss: 1.7092 - val_accuracy: 0.6135 Epoch 11/100 1148/1148 [==============================] - 10s 9ms/step - loss: 1.3910 - accuracy: 0.6588 - val_loss: 1.6626 - val_accuracy: 0.6246 Epoch 12/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3336 - accuracy: 0.6735 - val_loss: 1.6190 - val_accuracy: 0.6342 Epoch 13/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2704 - accuracy: 0.6887 - val_loss: 1.5918 - val_accuracy: 0.6356 Epoch 14/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2236 - accuracy: 0.6959 - val_loss: 1.5606 - val_accuracy: 0.6414 Epoch 15/100 1148/1148 [==============================] - 10s 8ms/step - loss: 1.1919 - accuracy: 0.7019 - val_loss: 1.5484 - val_accuracy: 0.6480 Epoch 16/100 1148/1148 [==============================] - 10s 8ms/step - loss: 1.1446 - accuracy: 0.7106 - val_loss: 1.5203 - val_accuracy: 0.6514 Epoch 17/100 1148/1148 [==============================] - 10s 9ms/step - loss: 1.1136 - accuracy: 0.7187 - val_loss: 1.5128 - val_accuracy: 0.6595 Epoch 18/100 1148/1148 [==============================] - 10s 8ms/step - loss: 1.0865 - accuracy: 0.7239 - val_loss: 1.4978 - val_accuracy: 0.6606 Epoch 19/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0612 - accuracy: 0.7301 - val_loss: 1.4906 - val_accuracy: 0.6582 Epoch 20/100 1148/1148 [==============================] - 10s 8ms/step - loss: 1.0394 - accuracy: 0.7348 - val_loss: 1.4774 - val_accuracy: 0.6633 Epoch 21/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0127 - accuracy: 0.7397 - val_loss: 1.4735 - val_accuracy: 0.6698 Epoch 22/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0028 - accuracy: 0.7421 - val_loss: 1.4574 - val_accuracy: 0.6729 Epoch 23/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9809 - accuracy: 0.7461 - val_loss: 1.4603 - val_accuracy: 0.6719 Epoch 24/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9666 - accuracy: 0.7503 - val_loss: 1.4512 - val_accuracy: 0.6723 Epoch 25/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9534 - accuracy: 0.7517 - val_loss: 1.4570 - val_accuracy: 0.6751 Epoch 26/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9407 - accuracy: 0.7541 - val_loss: 1.4513 - val_accuracy: 0.6729 Epoch 27/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9285 - accuracy: 0.7579 - val_loss: 1.4549 - val_accuracy: 0.6768 Epoch 28/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9136 - accuracy: 0.7615 - val_loss: 1.4533 - val_accuracy: 0.6735 Epoch 29/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9145 - accuracy: 0.7614 - val_loss: 1.4424 - val_accuracy: 0.6818 Epoch 30/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8969 - accuracy: 0.7648 - val_loss: 1.4431 - val_accuracy: 0.6806 Epoch 31/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8911 - accuracy: 0.7633 - val_loss: 1.4436 - val_accuracy: 0.6773 Epoch 32/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8833 - accuracy: 0.7669 - val_loss: 1.4474 - val_accuracy: 0.6816 Epoch 33/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8721 - accuracy: 0.7682 - val_loss: 1.4367 - val_accuracy: 0.6834 Epoch 34/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8659 - accuracy: 0.7696 - val_loss: 1.4392 - val_accuracy: 0.6791 Epoch 35/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8643 - accuracy: 0.7695 - val_loss: 1.4560 - val_accuracy: 0.6800 Epoch 36/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8597 - accuracy: 0.7716 - val_loss: 1.4467 - val_accuracy: 0.6825 Epoch 37/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8528 - accuracy: 0.7736 - val_loss: 1.4553 - val_accuracy: 0.6800 Epoch 38/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8461 - accuracy: 0.7738 - val_loss: 1.4443 - val_accuracy: 0.6827 Epoch 39/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8376 - accuracy: 0.7771 - val_loss: 1.4517 - val_accuracy: 0.6826 Epoch 40/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8376 - accuracy: 0.7761 - val_loss: 1.4498 - val_accuracy: 0.6852 Epoch 41/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8413 - accuracy: 0.7751 - val_loss: 1.4545 - val_accuracy: 0.6822 Epoch 42/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8260 - accuracy: 0.7795 - val_loss: 1.4478 - val_accuracy: 0.6836 Epoch 43/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8254 - accuracy: 0.7769 - val_loss: 1.4482 - val_accuracy: 0.6839 Epoch 44/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8206 - accuracy: 0.7800 - val_loss: 1.4297 - val_accuracy: 0.6854 Epoch 45/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8193 - accuracy: 0.7784 - val_loss: 1.4527 - val_accuracy: 0.6885 Epoch 46/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8113 - accuracy: 0.7798 - val_loss: 1.4471 - val_accuracy: 0.6877 Epoch 47/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8082 - accuracy: 0.7791 - val_loss: 1.4713 - val_accuracy: 0.6837 Epoch 48/100 1148/1148 [==============================] - 10s 9ms/step - loss: 0.8105 - accuracy: 0.7813 - val_loss: 1.4616 - val_accuracy: 0.6863 Epoch 49/100 1148/1148 [==============================] - 10s 9ms/step - loss: 0.8061 - accuracy: 0.7813 - val_loss: 1.4560 - val_accuracy: 0.6844 Epoch 50/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8051 - accuracy: 0.7809 - val_loss: 1.4525 - val_accuracy: 0.6891 Epoch 51/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7989 - accuracy: 0.7816 - val_loss: 1.4526 - val_accuracy: 0.6904 Epoch 52/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7953 - accuracy: 0.7826 - val_loss: 1.4635 - val_accuracy: 0.6885 Epoch 53/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7941 - accuracy: 0.7829 - val_loss: 1.4634 - val_accuracy: 0.6886 Epoch 54/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7915 - accuracy: 0.7850 - val_loss: 1.4598 - val_accuracy: 0.6898 Epoch 55/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7870 - accuracy: 0.7842 - val_loss: 1.4586 - val_accuracy: 0.6863 Epoch 56/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7859 - accuracy: 0.7861 - val_loss: 1.4539 - val_accuracy: 0.6923 Epoch 57/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7879 - accuracy: 0.7832 - val_loss: 1.4700 - val_accuracy: 0.6915 Epoch 58/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7863 - accuracy: 0.7842 - val_loss: 1.4552 - val_accuracy: 0.6906 Epoch 59/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7806 - accuracy: 0.7839 - val_loss: 1.4640 - val_accuracy: 0.6888 Epoch 60/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7846 - accuracy: 0.7836 - val_loss: 1.4529 - val_accuracy: 0.6921 Epoch 61/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7802 - accuracy: 0.7844 - val_loss: 1.4668 - val_accuracy: 0.6926 Epoch 62/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7711 - accuracy: 0.7870 - val_loss: 1.4690 - val_accuracy: 0.6891 Epoch 63/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7759 - accuracy: 0.7854 - val_loss: 1.4560 - val_accuracy: 0.6926 Epoch 64/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7766 - accuracy: 0.7853 - val_loss: 1.4585 - val_accuracy: 0.6902 Epoch 65/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7696 - accuracy: 0.7873 - val_loss: 1.4720 - val_accuracy: 0.6903 Epoch 66/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7695 - accuracy: 0.7860 - val_loss: 1.4538 - val_accuracy: 0.6939 Epoch 67/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7669 - accuracy: 0.7863 - val_loss: 1.4627 - val_accuracy: 0.6952 Epoch 68/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7673 - accuracy: 0.7865 - val_loss: 1.4728 - val_accuracy: 0.6884 Epoch 69/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7665 - accuracy: 0.7879 - val_loss: 1.4806 - val_accuracy: 0.6922 Epoch 70/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7659 - accuracy: 0.7873 - val_loss: 1.4692 - val_accuracy: 0.6922 Epoch 71/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7603 - accuracy: 0.7881 - val_loss: 1.4600 - val_accuracy: 0.6905 Epoch 72/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7622 - accuracy: 0.7877 - val_loss: 1.4578 - val_accuracy: 0.6939 Epoch 73/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7627 - accuracy: 0.7876 - val_loss: 1.4651 - val_accuracy: 0.6921 Epoch 74/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7594 - accuracy: 0.7883 - val_loss: 1.4824 - val_accuracy: 0.6923 Epoch 75/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7558 - accuracy: 0.7893 - val_loss: 1.4756 - val_accuracy: 0.6938 Epoch 76/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7598 - accuracy: 0.7879 - val_loss: 1.4757 - val_accuracy: 0.6906 Epoch 77/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7581 - accuracy: 0.7885 - val_loss: 1.4643 - val_accuracy: 0.6955 Epoch 78/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7532 - accuracy: 0.7890 - val_loss: 1.4609 - val_accuracy: 0.6928 Epoch 79/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7552 - accuracy: 0.7888 - val_loss: 1.4830 - val_accuracy: 0.6919 Epoch 80/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7572 - accuracy: 0.7872 - val_loss: 1.4604 - val_accuracy: 0.6958 Epoch 81/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7489 - accuracy: 0.7894 - val_loss: 1.4694 - val_accuracy: 0.6940 Epoch 82/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7511 - accuracy: 0.7879 - val_loss: 1.4668 - val_accuracy: 0.6937 Epoch 83/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7499 - accuracy: 0.7883 - val_loss: 1.4522 - val_accuracy: 0.6935 Epoch 84/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7493 - accuracy: 0.7885 - val_loss: 1.4683 - val_accuracy: 0.6909 Epoch 85/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7454 - accuracy: 0.7910 - val_loss: 1.4847 - val_accuracy: 0.6918 Epoch 86/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7414 - accuracy: 0.7910 - val_loss: 1.4866 - val_accuracy: 0.6929 Epoch 87/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7423 - accuracy: 0.7893 - val_loss: 1.4869 - val_accuracy: 0.6924 Epoch 88/100 1148/1148 [==============================] - 10s 9ms/step - loss: 0.7488 - accuracy: 0.7901 - val_loss: 1.4677 - val_accuracy: 0.6920 Epoch 89/100 1148/1148 [==============================] - 10s 9ms/step - loss: 0.7379 - accuracy: 0.7909 - val_loss: 1.4956 - val_accuracy: 0.6931 Epoch 90/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7455 - accuracy: 0.7877 - val_loss: 1.4773 - val_accuracy: 0.6935 Epoch 91/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7413 - accuracy: 0.7906 - val_loss: 1.4820 - val_accuracy: 0.6934 Epoch 92/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7444 - accuracy: 0.7883 - val_loss: 1.4549 - val_accuracy: 0.6954 Epoch 93/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7421 - accuracy: 0.7895 - val_loss: 1.4668 - val_accuracy: 0.6952 Epoch 94/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7400 - accuracy: 0.7904 - val_loss: 1.4743 - val_accuracy: 0.6943 Epoch 95/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7374 - accuracy: 0.7903 - val_loss: 1.4709 - val_accuracy: 0.6982 Epoch 96/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7397 - accuracy: 0.7922 - val_loss: 1.4779 - val_accuracy: 0.6936 Epoch 97/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7371 - accuracy: 0.7900 - val_loss: 1.4869 - val_accuracy: 0.6944 Epoch 98/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7362 - accuracy: 0.7921 - val_loss: 1.4985 - val_accuracy: 0.6939 Epoch 99/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7326 - accuracy: 0.7909 - val_loss: 1.4931 - val_accuracy: 0.6943 Epoch 100/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7344 - accuracy: 0.7919 - val_loss: 1.4745 - val_accuracy: 0.6966
Bi_LSTM_V1.summary()
Model: "bi_directional_lstm_v1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 512) 546816
l)
dropout (Dropout) (None, 512) 0
dense (Dense) (None, 1199) 615087
=================================================================
Total params: 1,173,893
Trainable params: 1,173,893
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_LSTM_V1_history.history)
Observations
Bi_LSTM_V1.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 4ms/step - loss: 1.4729 - accuracy: 0.6929
[1.472949743270874, 0.692873477935791]
Observations
tf.keras.backend.clear_session()
# Create the model
Bi_LSTM_V3 = Sequential(
name='bi_directional_lstm_v3',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(LSTM(128, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_LSTM_V3.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_LSTM_V3_history = Bi_LSTM_V3.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 11s 8ms/step - loss: 5.1227 - accuracy: 0.1152 - val_loss: 4.6069 - val_accuracy: 0.1430 Epoch 2/100 1148/1148 [==============================] - 9s 8ms/step - loss: 4.3421 - accuracy: 0.1775 - val_loss: 4.1059 - val_accuracy: 0.2076 Epoch 3/100 1148/1148 [==============================] - 9s 8ms/step - loss: 3.8228 - accuracy: 0.2440 - val_loss: 3.6392 - val_accuracy: 0.2763 Epoch 4/100 1148/1148 [==============================] - 9s 8ms/step - loss: 3.4152 - accuracy: 0.2981 - val_loss: 3.2881 - val_accuracy: 0.3258 Epoch 5/100 1148/1148 [==============================] - 9s 7ms/step - loss: 3.0972 - accuracy: 0.3418 - val_loss: 3.0152 - val_accuracy: 0.3680 Epoch 6/100 1148/1148 [==============================] - 9s 7ms/step - loss: 2.8305 - accuracy: 0.3816 - val_loss: 2.7948 - val_accuracy: 0.4029 Epoch 7/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.6061 - accuracy: 0.4192 - val_loss: 2.5926 - val_accuracy: 0.4483 Epoch 8/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.4178 - accuracy: 0.4538 - val_loss: 2.4287 - val_accuracy: 0.4720 Epoch 9/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.2460 - accuracy: 0.4854 - val_loss: 2.2788 - val_accuracy: 0.5037 Epoch 10/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.0983 - accuracy: 0.5134 - val_loss: 2.1502 - val_accuracy: 0.5293 Epoch 11/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.9754 - accuracy: 0.5404 - val_loss: 2.0421 - val_accuracy: 0.5505 Epoch 12/100 1148/1148 [==============================] - 10s 9ms/step - loss: 1.8599 - accuracy: 0.5597 - val_loss: 1.9719 - val_accuracy: 0.5648 Epoch 13/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.7601 - accuracy: 0.5786 - val_loss: 1.8841 - val_accuracy: 0.5812 Epoch 14/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.6789 - accuracy: 0.5938 - val_loss: 1.8129 - val_accuracy: 0.5898 Epoch 15/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5998 - accuracy: 0.6080 - val_loss: 1.7419 - val_accuracy: 0.6033 Epoch 16/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5263 - accuracy: 0.6251 - val_loss: 1.6932 - val_accuracy: 0.6164 Epoch 17/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4673 - accuracy: 0.6371 - val_loss: 1.6521 - val_accuracy: 0.6265 Epoch 18/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4124 - accuracy: 0.6490 - val_loss: 1.6168 - val_accuracy: 0.6344 Epoch 19/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3664 - accuracy: 0.6576 - val_loss: 1.6140 - val_accuracy: 0.6303 Epoch 20/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.3219 - accuracy: 0.6676 - val_loss: 1.5446 - val_accuracy: 0.6452 Epoch 21/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2835 - accuracy: 0.6764 - val_loss: 1.5321 - val_accuracy: 0.6483 Epoch 22/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2476 - accuracy: 0.6812 - val_loss: 1.5077 - val_accuracy: 0.6581 Epoch 23/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.2205 - accuracy: 0.6870 - val_loss: 1.4848 - val_accuracy: 0.6621 Epoch 24/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1964 - accuracy: 0.6937 - val_loss: 1.4638 - val_accuracy: 0.6626 Epoch 25/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.1676 - accuracy: 0.7008 - val_loss: 1.4509 - val_accuracy: 0.6679 Epoch 26/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1440 - accuracy: 0.7053 - val_loss: 1.4479 - val_accuracy: 0.6672 Epoch 27/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1163 - accuracy: 0.7110 - val_loss: 1.4389 - val_accuracy: 0.6725 Epoch 28/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0997 - accuracy: 0.7136 - val_loss: 1.4103 - val_accuracy: 0.6735 Epoch 29/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0903 - accuracy: 0.7164 - val_loss: 1.4017 - val_accuracy: 0.6779 Epoch 30/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0689 - accuracy: 0.7209 - val_loss: 1.4075 - val_accuracy: 0.6780 Epoch 31/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0503 - accuracy: 0.7239 - val_loss: 1.3746 - val_accuracy: 0.6847 Epoch 32/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0444 - accuracy: 0.7255 - val_loss: 1.4021 - val_accuracy: 0.6764 Epoch 33/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0236 - accuracy: 0.7326 - val_loss: 1.3929 - val_accuracy: 0.6772 Epoch 34/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0184 - accuracy: 0.7318 - val_loss: 1.3804 - val_accuracy: 0.6832 Epoch 35/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0011 - accuracy: 0.7358 - val_loss: 1.3688 - val_accuracy: 0.6887 Epoch 36/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9905 - accuracy: 0.7364 - val_loss: 1.3696 - val_accuracy: 0.6852 Epoch 37/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9830 - accuracy: 0.7376 - val_loss: 1.3660 - val_accuracy: 0.6871 Epoch 38/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9733 - accuracy: 0.7401 - val_loss: 1.3646 - val_accuracy: 0.6877 Epoch 39/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9672 - accuracy: 0.7422 - val_loss: 1.3617 - val_accuracy: 0.6868 Epoch 40/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9598 - accuracy: 0.7451 - val_loss: 1.3583 - val_accuracy: 0.6915 Epoch 41/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9475 - accuracy: 0.7479 - val_loss: 1.3504 - val_accuracy: 0.6924 Epoch 42/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9378 - accuracy: 0.7488 - val_loss: 1.3620 - val_accuracy: 0.6912 Epoch 43/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9289 - accuracy: 0.7523 - val_loss: 1.3402 - val_accuracy: 0.6976 Epoch 44/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9281 - accuracy: 0.7517 - val_loss: 1.3490 - val_accuracy: 0.6938 Epoch 45/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9207 - accuracy: 0.7512 - val_loss: 1.3487 - val_accuracy: 0.6979 Epoch 46/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9128 - accuracy: 0.7545 - val_loss: 1.3435 - val_accuracy: 0.6959 Epoch 47/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9006 - accuracy: 0.7575 - val_loss: 1.3429 - val_accuracy: 0.6952 Epoch 48/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8994 - accuracy: 0.7567 - val_loss: 1.3482 - val_accuracy: 0.6963 Epoch 49/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8882 - accuracy: 0.7587 - val_loss: 1.3597 - val_accuracy: 0.6970 Epoch 50/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8894 - accuracy: 0.7586 - val_loss: 1.3487 - val_accuracy: 0.6982 Epoch 51/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8839 - accuracy: 0.7591 - val_loss: 1.3392 - val_accuracy: 0.7014 Epoch 52/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8790 - accuracy: 0.7608 - val_loss: 1.3588 - val_accuracy: 0.6981 Epoch 53/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8714 - accuracy: 0.7633 - val_loss: 1.3453 - val_accuracy: 0.6993 Epoch 54/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8715 - accuracy: 0.7637 - val_loss: 1.3444 - val_accuracy: 0.6985 Epoch 55/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8719 - accuracy: 0.7626 - val_loss: 1.3478 - val_accuracy: 0.6975 Epoch 56/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8622 - accuracy: 0.7633 - val_loss: 1.3531 - val_accuracy: 0.6988 Epoch 57/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8529 - accuracy: 0.7670 - val_loss: 1.3449 - val_accuracy: 0.7006 Epoch 58/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8539 - accuracy: 0.7665 - val_loss: 1.3457 - val_accuracy: 0.7016 Epoch 59/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8511 - accuracy: 0.7661 - val_loss: 1.3534 - val_accuracy: 0.7008 Epoch 60/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8484 - accuracy: 0.7688 - val_loss: 1.3642 - val_accuracy: 0.7002 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8453 - accuracy: 0.7688 - val_loss: 1.3606 - val_accuracy: 0.6988 Epoch 62/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8444 - accuracy: 0.7681 - val_loss: 1.3501 - val_accuracy: 0.7005 Epoch 63/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8411 - accuracy: 0.7682 - val_loss: 1.3451 - val_accuracy: 0.7020 Epoch 64/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8299 - accuracy: 0.7709 - val_loss: 1.3659 - val_accuracy: 0.6992 Epoch 65/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8331 - accuracy: 0.7701 - val_loss: 1.3567 - val_accuracy: 0.7004 Epoch 66/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8330 - accuracy: 0.7701 - val_loss: 1.3563 - val_accuracy: 0.7002 Epoch 67/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8258 - accuracy: 0.7727 - val_loss: 1.3760 - val_accuracy: 0.6977 Epoch 68/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8271 - accuracy: 0.7704 - val_loss: 1.3594 - val_accuracy: 0.6993 Epoch 69/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8243 - accuracy: 0.7726 - val_loss: 1.3550 - val_accuracy: 0.7022 Epoch 70/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8242 - accuracy: 0.7720 - val_loss: 1.3658 - val_accuracy: 0.7039 Epoch 71/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8159 - accuracy: 0.7745 - val_loss: 1.3672 - val_accuracy: 0.7010 Epoch 72/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8176 - accuracy: 0.7732 - val_loss: 1.3743 - val_accuracy: 0.7063 Epoch 73/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8128 - accuracy: 0.7738 - val_loss: 1.3555 - val_accuracy: 0.7010 Epoch 74/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8092 - accuracy: 0.7742 - val_loss: 1.3623 - val_accuracy: 0.7032 Epoch 75/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8031 - accuracy: 0.7763 - val_loss: 1.3872 - val_accuracy: 0.7028 Epoch 76/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8088 - accuracy: 0.7773 - val_loss: 1.3688 - val_accuracy: 0.7035 Epoch 77/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8057 - accuracy: 0.7754 - val_loss: 1.3656 - val_accuracy: 0.7048 Epoch 78/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8024 - accuracy: 0.7748 - val_loss: 1.3724 - val_accuracy: 0.7026 Epoch 79/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8054 - accuracy: 0.7765 - val_loss: 1.3717 - val_accuracy: 0.7027 Epoch 80/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7978 - accuracy: 0.7779 - val_loss: 1.3791 - val_accuracy: 0.7010 Epoch 81/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7968 - accuracy: 0.7778 - val_loss: 1.3735 - val_accuracy: 0.7038 Epoch 82/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7960 - accuracy: 0.7786 - val_loss: 1.3714 - val_accuracy: 0.7052 Epoch 83/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7901 - accuracy: 0.7777 - val_loss: 1.3909 - val_accuracy: 0.7018 Epoch 84/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7920 - accuracy: 0.7792 - val_loss: 1.3779 - val_accuracy: 0.7051 Epoch 85/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7883 - accuracy: 0.7805 - val_loss: 1.3778 - val_accuracy: 0.7018 Epoch 86/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7936 - accuracy: 0.7768 - val_loss: 1.3842 - val_accuracy: 0.7047 Epoch 87/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7881 - accuracy: 0.7807 - val_loss: 1.3802 - val_accuracy: 0.7027 Epoch 88/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7827 - accuracy: 0.7803 - val_loss: 1.3771 - val_accuracy: 0.7046 Epoch 89/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7826 - accuracy: 0.7796 - val_loss: 1.3880 - val_accuracy: 0.7070 Epoch 90/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7803 - accuracy: 0.7814 - val_loss: 1.3844 - val_accuracy: 0.7018 Epoch 91/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7831 - accuracy: 0.7818 - val_loss: 1.3830 - val_accuracy: 0.7024 Epoch 92/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7807 - accuracy: 0.7795 - val_loss: 1.3801 - val_accuracy: 0.7051 Epoch 93/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7755 - accuracy: 0.7805 - val_loss: 1.3740 - val_accuracy: 0.7051 Epoch 94/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7749 - accuracy: 0.7811 - val_loss: 1.3826 - val_accuracy: 0.7031 Epoch 95/100 1148/1148 [==============================] - 10s 9ms/step - loss: 0.7764 - accuracy: 0.7813 - val_loss: 1.3784 - val_accuracy: 0.7021 Epoch 96/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7746 - accuracy: 0.7816 - val_loss: 1.3870 - val_accuracy: 0.7033 Epoch 97/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7782 - accuracy: 0.7815 - val_loss: 1.3817 - val_accuracy: 0.7055 Epoch 98/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7689 - accuracy: 0.7824 - val_loss: 1.3994 - val_accuracy: 0.7020 Epoch 99/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7708 - accuracy: 0.7820 - val_loss: 1.3918 - val_accuracy: 0.7064 Epoch 100/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7701 - accuracy: 0.7832 - val_loss: 1.3850 - val_accuracy: 0.7013
Bi_LSTM_V3.summary()
Model: "bi_directional_lstm_v3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 256) 142336
l)
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 462,469
Trainable params: 462,469
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_LSTM_V3_history.history)
Observations
Bi_LSTM_V3.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.3746 - accuracy: 0.7029
[1.374636173248291, 0.7029258012771606]
Observations
tf.keras.backend.clear_session()
# Create the model
Bi_LSTM_V4 = Sequential(
name='bi_directional_lstm_v4',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(LSTM(64, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_LSTM_V4.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_LSTM_V4_history = Bi_LSTM_V4.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 10s 7ms/step - loss: 5.2180 - accuracy: 0.1085 - val_loss: 4.7985 - val_accuracy: 0.1368 Epoch 2/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.5783 - accuracy: 0.1602 - val_loss: 4.3700 - val_accuracy: 0.1930 Epoch 3/100 1148/1148 [==============================] - 8s 7ms/step - loss: 4.1695 - accuracy: 0.2143 - val_loss: 4.0097 - val_accuracy: 0.2355 Epoch 4/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.8603 - accuracy: 0.2546 - val_loss: 3.7258 - val_accuracy: 0.2786 Epoch 5/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.6080 - accuracy: 0.2857 - val_loss: 3.4999 - val_accuracy: 0.3151 Epoch 6/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.3989 - accuracy: 0.3139 - val_loss: 3.3095 - val_accuracy: 0.3397 Epoch 7/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.2186 - accuracy: 0.3372 - val_loss: 3.1428 - val_accuracy: 0.3680 Epoch 8/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.0578 - accuracy: 0.3558 - val_loss: 2.9932 - val_accuracy: 0.3883 Epoch 9/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.9204 - accuracy: 0.3762 - val_loss: 2.8587 - val_accuracy: 0.4083 Epoch 10/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.7978 - accuracy: 0.3932 - val_loss: 2.7527 - val_accuracy: 0.4221 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.6886 - accuracy: 0.4086 - val_loss: 2.6570 - val_accuracy: 0.4362 Epoch 12/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.5978 - accuracy: 0.4246 - val_loss: 2.5716 - val_accuracy: 0.4541 Epoch 13/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.5139 - accuracy: 0.4352 - val_loss: 2.5003 - val_accuracy: 0.4607 Epoch 14/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.4342 - accuracy: 0.4468 - val_loss: 2.4368 - val_accuracy: 0.4658 Epoch 15/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3792 - accuracy: 0.4562 - val_loss: 2.3674 - val_accuracy: 0.4814 Epoch 16/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.3038 - accuracy: 0.4703 - val_loss: 2.3113 - val_accuracy: 0.4898 Epoch 17/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.2587 - accuracy: 0.4757 - val_loss: 2.2603 - val_accuracy: 0.4987 Epoch 18/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.2020 - accuracy: 0.4856 - val_loss: 2.2147 - val_accuracy: 0.5071 Epoch 19/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1550 - accuracy: 0.4909 - val_loss: 2.1714 - val_accuracy: 0.5186 Epoch 20/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1162 - accuracy: 0.4981 - val_loss: 2.1323 - val_accuracy: 0.5268 Epoch 21/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.0736 - accuracy: 0.5059 - val_loss: 2.1032 - val_accuracy: 0.5329 Epoch 22/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.0329 - accuracy: 0.5111 - val_loss: 2.0576 - val_accuracy: 0.5423 Epoch 23/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9973 - accuracy: 0.5207 - val_loss: 2.0238 - val_accuracy: 0.5483 Epoch 24/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9611 - accuracy: 0.5248 - val_loss: 1.9973 - val_accuracy: 0.5521 Epoch 25/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9385 - accuracy: 0.5322 - val_loss: 1.9671 - val_accuracy: 0.5575 Epoch 26/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9050 - accuracy: 0.5360 - val_loss: 1.9433 - val_accuracy: 0.5620 Epoch 27/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8760 - accuracy: 0.5434 - val_loss: 1.9151 - val_accuracy: 0.5691 Epoch 28/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8537 - accuracy: 0.5449 - val_loss: 1.8977 - val_accuracy: 0.5734 Epoch 29/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8250 - accuracy: 0.5525 - val_loss: 1.8761 - val_accuracy: 0.5765 Epoch 30/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8054 - accuracy: 0.5556 - val_loss: 1.8560 - val_accuracy: 0.5821 Epoch 31/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7835 - accuracy: 0.5598 - val_loss: 1.8344 - val_accuracy: 0.5805 Epoch 32/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7588 - accuracy: 0.5652 - val_loss: 1.8157 - val_accuracy: 0.5832 Epoch 33/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7336 - accuracy: 0.5717 - val_loss: 1.7923 - val_accuracy: 0.5933 Epoch 34/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7095 - accuracy: 0.5721 - val_loss: 1.7841 - val_accuracy: 0.5948 Epoch 35/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6842 - accuracy: 0.5800 - val_loss: 1.7678 - val_accuracy: 0.5965 Epoch 36/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6668 - accuracy: 0.5839 - val_loss: 1.7490 - val_accuracy: 0.6016 Epoch 37/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6397 - accuracy: 0.5904 - val_loss: 1.7296 - val_accuracy: 0.6059 Epoch 38/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6291 - accuracy: 0.5918 - val_loss: 1.7163 - val_accuracy: 0.6087 Epoch 39/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5985 - accuracy: 0.5960 - val_loss: 1.6951 - val_accuracy: 0.6157 Epoch 40/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5844 - accuracy: 0.6027 - val_loss: 1.6860 - val_accuracy: 0.6174 Epoch 41/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5671 - accuracy: 0.6069 - val_loss: 1.6700 - val_accuracy: 0.6207 Epoch 42/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5492 - accuracy: 0.6087 - val_loss: 1.6657 - val_accuracy: 0.6201 Epoch 43/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5311 - accuracy: 0.6118 - val_loss: 1.6514 - val_accuracy: 0.6210 Epoch 44/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5141 - accuracy: 0.6163 - val_loss: 1.6489 - val_accuracy: 0.6243 Epoch 45/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4966 - accuracy: 0.6187 - val_loss: 1.6405 - val_accuracy: 0.6246 Epoch 46/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4868 - accuracy: 0.6221 - val_loss: 1.6264 - val_accuracy: 0.6296 Epoch 47/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4648 - accuracy: 0.6265 - val_loss: 1.6080 - val_accuracy: 0.6353 Epoch 48/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4535 - accuracy: 0.6266 - val_loss: 1.5990 - val_accuracy: 0.6367 Epoch 49/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4367 - accuracy: 0.6321 - val_loss: 1.5905 - val_accuracy: 0.6373 Epoch 50/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4264 - accuracy: 0.6344 - val_loss: 1.5848 - val_accuracy: 0.6413 Epoch 51/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4127 - accuracy: 0.6369 - val_loss: 1.5809 - val_accuracy: 0.6422 Epoch 52/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3979 - accuracy: 0.6395 - val_loss: 1.5791 - val_accuracy: 0.6411 Epoch 53/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3885 - accuracy: 0.6425 - val_loss: 1.5593 - val_accuracy: 0.6472 Epoch 54/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3698 - accuracy: 0.6447 - val_loss: 1.5558 - val_accuracy: 0.6467 Epoch 55/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3672 - accuracy: 0.6452 - val_loss: 1.5466 - val_accuracy: 0.6500 Epoch 56/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3441 - accuracy: 0.6508 - val_loss: 1.5430 - val_accuracy: 0.6514 Epoch 57/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3408 - accuracy: 0.6524 - val_loss: 1.5286 - val_accuracy: 0.6562 Epoch 58/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3325 - accuracy: 0.6537 - val_loss: 1.5345 - val_accuracy: 0.6568 Epoch 59/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3186 - accuracy: 0.6576 - val_loss: 1.5241 - val_accuracy: 0.6552 Epoch 60/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3183 - accuracy: 0.6576 - val_loss: 1.5162 - val_accuracy: 0.6550 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3032 - accuracy: 0.6605 - val_loss: 1.5154 - val_accuracy: 0.6602 Epoch 62/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2979 - accuracy: 0.6632 - val_loss: 1.5102 - val_accuracy: 0.6603 Epoch 63/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2825 - accuracy: 0.6667 - val_loss: 1.4961 - val_accuracy: 0.6645 Epoch 64/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2787 - accuracy: 0.6668 - val_loss: 1.5047 - val_accuracy: 0.6629 Epoch 65/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2690 - accuracy: 0.6662 - val_loss: 1.4908 - val_accuracy: 0.6623 Epoch 66/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2609 - accuracy: 0.6731 - val_loss: 1.4859 - val_accuracy: 0.6671 Epoch 67/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2440 - accuracy: 0.6740 - val_loss: 1.4859 - val_accuracy: 0.6661 Epoch 68/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2393 - accuracy: 0.6744 - val_loss: 1.4900 - val_accuracy: 0.6657 Epoch 69/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2357 - accuracy: 0.6745 - val_loss: 1.4842 - val_accuracy: 0.6679 Epoch 70/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2300 - accuracy: 0.6777 - val_loss: 1.4820 - val_accuracy: 0.6684 Epoch 71/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2257 - accuracy: 0.6776 - val_loss: 1.4711 - val_accuracy: 0.6712 Epoch 72/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2102 - accuracy: 0.6792 - val_loss: 1.4670 - val_accuracy: 0.6710 Epoch 73/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2042 - accuracy: 0.6800 - val_loss: 1.4646 - val_accuracy: 0.6737 Epoch 74/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2030 - accuracy: 0.6821 - val_loss: 1.4753 - val_accuracy: 0.6691 Epoch 75/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1934 - accuracy: 0.6851 - val_loss: 1.4762 - val_accuracy: 0.6719 Epoch 76/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1805 - accuracy: 0.6881 - val_loss: 1.4626 - val_accuracy: 0.6738 Epoch 77/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1899 - accuracy: 0.6839 - val_loss: 1.4699 - val_accuracy: 0.6737 Epoch 78/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1801 - accuracy: 0.6867 - val_loss: 1.4604 - val_accuracy: 0.6756 Epoch 79/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1700 - accuracy: 0.6910 - val_loss: 1.4513 - val_accuracy: 0.6742 Epoch 80/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1652 - accuracy: 0.6919 - val_loss: 1.4513 - val_accuracy: 0.6770 Epoch 81/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1641 - accuracy: 0.6915 - val_loss: 1.4524 - val_accuracy: 0.6784 Epoch 82/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1521 - accuracy: 0.6937 - val_loss: 1.4616 - val_accuracy: 0.6765 Epoch 83/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1540 - accuracy: 0.6928 - val_loss: 1.4547 - val_accuracy: 0.6782 Epoch 84/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1452 - accuracy: 0.6964 - val_loss: 1.4529 - val_accuracy: 0.6775 Epoch 85/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1401 - accuracy: 0.6966 - val_loss: 1.4508 - val_accuracy: 0.6809 Epoch 86/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1320 - accuracy: 0.7012 - val_loss: 1.4469 - val_accuracy: 0.6830 Epoch 87/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1306 - accuracy: 0.6994 - val_loss: 1.4404 - val_accuracy: 0.6817 Epoch 88/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1204 - accuracy: 0.7034 - val_loss: 1.4426 - val_accuracy: 0.6863 Epoch 89/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1202 - accuracy: 0.7026 - val_loss: 1.4421 - val_accuracy: 0.6858 Epoch 90/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1190 - accuracy: 0.7045 - val_loss: 1.4424 - val_accuracy: 0.6820 Epoch 91/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1068 - accuracy: 0.7054 - val_loss: 1.4350 - val_accuracy: 0.6839 Epoch 92/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1176 - accuracy: 0.7015 - val_loss: 1.4400 - val_accuracy: 0.6822 Epoch 93/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0958 - accuracy: 0.7060 - val_loss: 1.4350 - val_accuracy: 0.6858 Epoch 94/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0940 - accuracy: 0.7065 - val_loss: 1.4285 - val_accuracy: 0.6845 Epoch 95/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0849 - accuracy: 0.7095 - val_loss: 1.4322 - val_accuracy: 0.6885 Epoch 96/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0870 - accuracy: 0.7110 - val_loss: 1.4485 - val_accuracy: 0.6845 Epoch 97/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0845 - accuracy: 0.7078 - val_loss: 1.4436 - val_accuracy: 0.6872 Epoch 98/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0866 - accuracy: 0.7101 - val_loss: 1.4341 - val_accuracy: 0.6869 Epoch 99/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0724 - accuracy: 0.7119 - val_loss: 1.4327 - val_accuracy: 0.6904 Epoch 100/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0652 - accuracy: 0.7136 - val_loss: 1.4334 - val_accuracy: 0.6885
Bi_LSTM_V4.summary()
Model: "bi_directional_lstm_v4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 128) 38400
l)
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 205,061
Trainable params: 205,061
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_LSTM_V4_history.history)
Observations
Bi_LSTM_V4.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.4263 - accuracy: 0.6929
[1.4263392686843872, 0.692873477935791]
Observations
tf.keras.backend.clear_session()
# Create the model
Bi_GRU_V1 = Sequential(
name='bi_directional_gru_v1',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(GRU(256, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_GRU_V1.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_GRU_V1_history = Bi_GRU_V1.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 11s 8ms/step - loss: 4.6431 - accuracy: 0.1840 - val_loss: 3.7274 - val_accuracy: 0.2822 Epoch 2/100 1148/1148 [==============================] - 9s 8ms/step - loss: 3.1970 - accuracy: 0.3538 - val_loss: 2.8156 - val_accuracy: 0.4192 Epoch 3/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.4855 - accuracy: 0.4655 - val_loss: 2.3670 - val_accuracy: 0.4951 Epoch 4/100 1148/1148 [==============================] - 9s 8ms/step - loss: 2.0936 - accuracy: 0.5310 - val_loss: 2.0878 - val_accuracy: 0.5485 Epoch 5/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.8338 - accuracy: 0.5793 - val_loss: 1.8986 - val_accuracy: 0.5829 Epoch 6/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.6488 - accuracy: 0.6145 - val_loss: 1.7937 - val_accuracy: 0.6054 Epoch 7/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.5234 - accuracy: 0.6394 - val_loss: 1.6942 - val_accuracy: 0.6213 Epoch 8/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.4213 - accuracy: 0.6584 - val_loss: 1.6261 - val_accuracy: 0.6330 Epoch 9/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.3388 - accuracy: 0.6768 - val_loss: 1.5711 - val_accuracy: 0.6490 Epoch 10/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.2691 - accuracy: 0.6922 - val_loss: 1.5304 - val_accuracy: 0.6578 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2148 - accuracy: 0.7019 - val_loss: 1.4939 - val_accuracy: 0.6645 Epoch 12/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1702 - accuracy: 0.7121 - val_loss: 1.4686 - val_accuracy: 0.6697 Epoch 13/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.1321 - accuracy: 0.7205 - val_loss: 1.4587 - val_accuracy: 0.6694 Epoch 14/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0968 - accuracy: 0.7252 - val_loss: 1.4359 - val_accuracy: 0.6762 Epoch 15/100 1148/1148 [==============================] - 9s 8ms/step - loss: 1.0716 - accuracy: 0.7303 - val_loss: 1.4297 - val_accuracy: 0.6760 Epoch 16/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0401 - accuracy: 0.7384 - val_loss: 1.4233 - val_accuracy: 0.6793 Epoch 17/100 1148/1148 [==============================] - 9s 7ms/step - loss: 1.0214 - accuracy: 0.7429 - val_loss: 1.4112 - val_accuracy: 0.6812 Epoch 18/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0006 - accuracy: 0.7459 - val_loss: 1.4127 - val_accuracy: 0.6856 Epoch 19/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.9912 - accuracy: 0.7467 - val_loss: 1.4033 - val_accuracy: 0.6883 Epoch 20/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9742 - accuracy: 0.7518 - val_loss: 1.3954 - val_accuracy: 0.6881 Epoch 21/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9572 - accuracy: 0.7540 - val_loss: 1.3982 - val_accuracy: 0.6894 Epoch 22/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9554 - accuracy: 0.7562 - val_loss: 1.3837 - val_accuracy: 0.6867 Epoch 23/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9344 - accuracy: 0.7594 - val_loss: 1.3903 - val_accuracy: 0.6907 Epoch 24/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9266 - accuracy: 0.7624 - val_loss: 1.3830 - val_accuracy: 0.6917 Epoch 25/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9169 - accuracy: 0.7625 - val_loss: 1.3947 - val_accuracy: 0.6898 Epoch 26/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9068 - accuracy: 0.7656 - val_loss: 1.3782 - val_accuracy: 0.6928 Epoch 27/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.9020 - accuracy: 0.7661 - val_loss: 1.3796 - val_accuracy: 0.6981 Epoch 28/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8970 - accuracy: 0.7660 - val_loss: 1.3879 - val_accuracy: 0.6963 Epoch 29/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8856 - accuracy: 0.7701 - val_loss: 1.3764 - val_accuracy: 0.6970 Epoch 30/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8848 - accuracy: 0.7673 - val_loss: 1.3769 - val_accuracy: 0.6973 Epoch 31/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8792 - accuracy: 0.7704 - val_loss: 1.3794 - val_accuracy: 0.6975 Epoch 32/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8817 - accuracy: 0.7687 - val_loss: 1.3890 - val_accuracy: 0.6991 Epoch 33/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8676 - accuracy: 0.7718 - val_loss: 1.3968 - val_accuracy: 0.6969 Epoch 34/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8652 - accuracy: 0.7723 - val_loss: 1.3829 - val_accuracy: 0.7006 Epoch 35/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8669 - accuracy: 0.7715 - val_loss: 1.3893 - val_accuracy: 0.7034 Epoch 36/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8592 - accuracy: 0.7729 - val_loss: 1.3912 - val_accuracy: 0.6991 Epoch 37/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8589 - accuracy: 0.7743 - val_loss: 1.3882 - val_accuracy: 0.6948 Epoch 38/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8478 - accuracy: 0.7747 - val_loss: 1.3793 - val_accuracy: 0.6976 Epoch 39/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8498 - accuracy: 0.7730 - val_loss: 1.4478 - val_accuracy: 0.6901 Epoch 40/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8495 - accuracy: 0.7766 - val_loss: 1.3870 - val_accuracy: 0.6997 Epoch 41/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8445 - accuracy: 0.7750 - val_loss: 1.3920 - val_accuracy: 0.7016 Epoch 42/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8422 - accuracy: 0.7768 - val_loss: 1.3935 - val_accuracy: 0.6998 Epoch 43/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8377 - accuracy: 0.7774 - val_loss: 1.4042 - val_accuracy: 0.7002 Epoch 44/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8418 - accuracy: 0.7758 - val_loss: 1.3900 - val_accuracy: 0.7001 Epoch 45/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8338 - accuracy: 0.7788 - val_loss: 1.3964 - val_accuracy: 0.6999 Epoch 46/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8345 - accuracy: 0.7770 - val_loss: 1.3926 - val_accuracy: 0.6999 Epoch 47/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8298 - accuracy: 0.7769 - val_loss: 1.3915 - val_accuracy: 0.7001 Epoch 48/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8290 - accuracy: 0.7785 - val_loss: 1.3963 - val_accuracy: 0.7020 Epoch 49/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8235 - accuracy: 0.7801 - val_loss: 1.3886 - val_accuracy: 0.7006 Epoch 50/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8228 - accuracy: 0.7790 - val_loss: 1.3995 - val_accuracy: 0.7005 Epoch 51/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8236 - accuracy: 0.7789 - val_loss: 1.3951 - val_accuracy: 0.7030 Epoch 52/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8229 - accuracy: 0.7779 - val_loss: 1.3902 - val_accuracy: 0.7033 Epoch 53/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8171 - accuracy: 0.7819 - val_loss: 1.3923 - val_accuracy: 0.7043 Epoch 54/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8178 - accuracy: 0.7816 - val_loss: 1.3928 - val_accuracy: 0.7032 Epoch 55/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8089 - accuracy: 0.7821 - val_loss: 1.3941 - val_accuracy: 0.7049 Epoch 56/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8167 - accuracy: 0.7806 - val_loss: 1.3996 - val_accuracy: 0.7006 Epoch 57/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8147 - accuracy: 0.7795 - val_loss: 1.4024 - val_accuracy: 0.7025 Epoch 58/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8109 - accuracy: 0.7801 - val_loss: 1.4087 - val_accuracy: 0.7048 Epoch 59/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8103 - accuracy: 0.7809 - val_loss: 1.4038 - val_accuracy: 0.7030 Epoch 60/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8074 - accuracy: 0.7815 - val_loss: 1.4015 - val_accuracy: 0.7059 Epoch 61/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8047 - accuracy: 0.7828 - val_loss: 1.4070 - val_accuracy: 0.7004 Epoch 62/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8045 - accuracy: 0.7831 - val_loss: 1.4059 - val_accuracy: 0.7046 Epoch 63/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8043 - accuracy: 0.7810 - val_loss: 1.4077 - val_accuracy: 0.7038 Epoch 64/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8033 - accuracy: 0.7827 - val_loss: 1.4133 - val_accuracy: 0.7026 Epoch 65/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8039 - accuracy: 0.7818 - val_loss: 1.4252 - val_accuracy: 0.7040 Epoch 66/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7971 - accuracy: 0.7841 - val_loss: 1.4078 - val_accuracy: 0.7062 Epoch 67/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.8005 - accuracy: 0.7826 - val_loss: 1.4063 - val_accuracy: 0.7040 Epoch 68/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8022 - accuracy: 0.7824 - val_loss: 1.4137 - val_accuracy: 0.7065 Epoch 69/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8000 - accuracy: 0.7823 - val_loss: 1.4164 - val_accuracy: 0.7008 Epoch 70/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7973 - accuracy: 0.7850 - val_loss: 1.4174 - val_accuracy: 0.7021 Epoch 71/100 1148/1148 [==============================] - 10s 8ms/step - loss: 0.7968 - accuracy: 0.7843 - val_loss: 1.4145 - val_accuracy: 0.7049 Epoch 72/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7927 - accuracy: 0.7849 - val_loss: 1.3990 - val_accuracy: 0.7064 Epoch 73/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7928 - accuracy: 0.7838 - val_loss: 1.4076 - val_accuracy: 0.7071 Epoch 74/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7915 - accuracy: 0.7842 - val_loss: 1.4145 - val_accuracy: 0.7051 Epoch 75/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7941 - accuracy: 0.7841 - val_loss: 1.3795 - val_accuracy: 0.7084 Epoch 76/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7924 - accuracy: 0.7834 - val_loss: 1.4101 - val_accuracy: 0.7053 Epoch 77/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7851 - accuracy: 0.7845 - val_loss: 1.4255 - val_accuracy: 0.7046 Epoch 78/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7881 - accuracy: 0.7833 - val_loss: 1.4294 - val_accuracy: 0.7056 Epoch 79/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7813 - accuracy: 0.7849 - val_loss: 1.4189 - val_accuracy: 0.7047 Epoch 80/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7859 - accuracy: 0.7837 - val_loss: 1.4177 - val_accuracy: 0.7044 Epoch 81/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7858 - accuracy: 0.7855 - val_loss: 1.4118 - val_accuracy: 0.7015 Epoch 82/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7820 - accuracy: 0.7843 - val_loss: 1.3887 - val_accuracy: 0.7080 Epoch 83/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7834 - accuracy: 0.7847 - val_loss: 1.4148 - val_accuracy: 0.7071 Epoch 84/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7809 - accuracy: 0.7870 - val_loss: 1.3977 - val_accuracy: 0.7061 Epoch 85/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7825 - accuracy: 0.7838 - val_loss: 1.3879 - val_accuracy: 0.7090 Epoch 86/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7773 - accuracy: 0.7871 - val_loss: 1.4177 - val_accuracy: 0.7062 Epoch 87/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7802 - accuracy: 0.7858 - val_loss: 1.4318 - val_accuracy: 0.7037 Epoch 88/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7810 - accuracy: 0.7854 - val_loss: 1.4070 - val_accuracy: 0.7042 Epoch 89/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7827 - accuracy: 0.7844 - val_loss: 1.3928 - val_accuracy: 0.7084 Epoch 90/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7805 - accuracy: 0.7863 - val_loss: 1.4067 - val_accuracy: 0.7049 Epoch 91/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7761 - accuracy: 0.7852 - val_loss: 1.4153 - val_accuracy: 0.7050 Epoch 92/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7792 - accuracy: 0.7847 - val_loss: 1.3885 - val_accuracy: 0.7072 Epoch 93/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.7778 - accuracy: 0.7853 - val_loss: 1.4208 - val_accuracy: 0.7042 Epoch 94/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7769 - accuracy: 0.7863 - val_loss: 1.4127 - val_accuracy: 0.7063 Epoch 95/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7739 - accuracy: 0.7867 - val_loss: 1.4163 - val_accuracy: 0.7059 Epoch 96/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7729 - accuracy: 0.7867 - val_loss: 1.4025 - val_accuracy: 0.7058 Epoch 97/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7718 - accuracy: 0.7864 - val_loss: 1.4080 - val_accuracy: 0.7063 Epoch 98/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7762 - accuracy: 0.7862 - val_loss: 1.4133 - val_accuracy: 0.7078 Epoch 99/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7703 - accuracy: 0.7889 - val_loss: 1.4212 - val_accuracy: 0.7051 Epoch 100/100 1148/1148 [==============================] - 9s 7ms/step - loss: 0.7747 - accuracy: 0.7856 - val_loss: 1.4124 - val_accuracy: 0.7085
Bi_GRU_V1.summary()
Model: "bi_directional_gru_v1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 512) 411648
l)
dropout (Dropout) (None, 512) 0
dense (Dense) (None, 1199) 615087
=================================================================
Total params: 1,038,725
Trainable params: 1,038,725
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_GRU_V1_history.history)
Observations
Bi_GRU_V1.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.4115 - accuracy: 0.7068
[1.4115307331085205, 0.7067669034004211]
Observations
tf.keras.backend.clear_session()
# Create the model
Bi_GRU_V2 = Sequential(
name='bi_directional_gru_v2',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(GRU(128, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_GRU_V2.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_GRU_V2_history = Bi_GRU_V2.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 10s 7ms/step - loss: 4.8441 - accuracy: 0.1580 - val_loss: 3.9480 - val_accuracy: 0.2610 Epoch 2/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.4911 - accuracy: 0.3145 - val_loss: 3.1225 - val_accuracy: 0.3746 Epoch 3/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.8281 - accuracy: 0.4083 - val_loss: 2.6272 - val_accuracy: 0.4536 Epoch 4/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.4069 - accuracy: 0.4777 - val_loss: 2.3292 - val_accuracy: 0.5059 Epoch 5/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.1382 - accuracy: 0.5211 - val_loss: 2.1166 - val_accuracy: 0.5426 Epoch 6/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9356 - accuracy: 0.5569 - val_loss: 1.9573 - val_accuracy: 0.5685 Epoch 7/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7927 - accuracy: 0.5833 - val_loss: 1.8417 - val_accuracy: 0.5912 Epoch 8/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6757 - accuracy: 0.6068 - val_loss: 1.7521 - val_accuracy: 0.6121 Epoch 9/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5808 - accuracy: 0.6249 - val_loss: 1.6853 - val_accuracy: 0.6240 Epoch 10/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5005 - accuracy: 0.6414 - val_loss: 1.6141 - val_accuracy: 0.6403 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4365 - accuracy: 0.6543 - val_loss: 1.5775 - val_accuracy: 0.6434 Epoch 12/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3717 - accuracy: 0.6706 - val_loss: 1.5438 - val_accuracy: 0.6520 Epoch 13/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3277 - accuracy: 0.6776 - val_loss: 1.5027 - val_accuracy: 0.6581 Epoch 14/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2841 - accuracy: 0.6855 - val_loss: 1.4789 - val_accuracy: 0.6666 Epoch 15/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2479 - accuracy: 0.6947 - val_loss: 1.4541 - val_accuracy: 0.6664 Epoch 16/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2118 - accuracy: 0.6998 - val_loss: 1.4252 - val_accuracy: 0.6751 Epoch 17/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1792 - accuracy: 0.7077 - val_loss: 1.4222 - val_accuracy: 0.6743 Epoch 18/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1534 - accuracy: 0.7128 - val_loss: 1.3964 - val_accuracy: 0.6792 Epoch 19/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1314 - accuracy: 0.7193 - val_loss: 1.3856 - val_accuracy: 0.6802 Epoch 20/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1154 - accuracy: 0.7205 - val_loss: 1.3823 - val_accuracy: 0.6848 Epoch 21/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0941 - accuracy: 0.7252 - val_loss: 1.3652 - val_accuracy: 0.6872 Epoch 22/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0804 - accuracy: 0.7278 - val_loss: 1.3569 - val_accuracy: 0.6883 Epoch 23/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0649 - accuracy: 0.7311 - val_loss: 1.3614 - val_accuracy: 0.6875 Epoch 24/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0393 - accuracy: 0.7356 - val_loss: 1.3468 - val_accuracy: 0.6945 Epoch 25/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0290 - accuracy: 0.7398 - val_loss: 1.3495 - val_accuracy: 0.6929 Epoch 26/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0221 - accuracy: 0.7400 - val_loss: 1.3480 - val_accuracy: 0.6885 Epoch 27/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0043 - accuracy: 0.7451 - val_loss: 1.3391 - val_accuracy: 0.6975 Epoch 28/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9945 - accuracy: 0.7445 - val_loss: 1.3306 - val_accuracy: 0.6968 Epoch 29/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9851 - accuracy: 0.7482 - val_loss: 1.3248 - val_accuracy: 0.6954 Epoch 30/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9740 - accuracy: 0.7485 - val_loss: 1.3276 - val_accuracy: 0.6988 Epoch 31/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9693 - accuracy: 0.7505 - val_loss: 1.3384 - val_accuracy: 0.6960 Epoch 32/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9583 - accuracy: 0.7531 - val_loss: 1.3288 - val_accuracy: 0.6981 Epoch 33/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9530 - accuracy: 0.7530 - val_loss: 1.3308 - val_accuracy: 0.6981 Epoch 34/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9403 - accuracy: 0.7557 - val_loss: 1.3349 - val_accuracy: 0.6961 Epoch 35/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9351 - accuracy: 0.7567 - val_loss: 1.3088 - val_accuracy: 0.7035 Epoch 36/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9357 - accuracy: 0.7588 - val_loss: 1.3203 - val_accuracy: 0.6969 Epoch 37/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9230 - accuracy: 0.7594 - val_loss: 1.3185 - val_accuracy: 0.7039 Epoch 38/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9147 - accuracy: 0.7619 - val_loss: 1.3225 - val_accuracy: 0.7033 Epoch 39/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9113 - accuracy: 0.7628 - val_loss: 1.3325 - val_accuracy: 0.7035 Epoch 40/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9104 - accuracy: 0.7640 - val_loss: 1.3197 - val_accuracy: 0.7007 Epoch 41/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9034 - accuracy: 0.7641 - val_loss: 1.3182 - val_accuracy: 0.7040 Epoch 42/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9000 - accuracy: 0.7646 - val_loss: 1.3182 - val_accuracy: 0.7059 Epoch 43/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8906 - accuracy: 0.7676 - val_loss: 1.3109 - val_accuracy: 0.7021 Epoch 44/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8890 - accuracy: 0.7671 - val_loss: 1.3159 - val_accuracy: 0.7043 Epoch 45/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8894 - accuracy: 0.7649 - val_loss: 1.3217 - val_accuracy: 0.7019 Epoch 46/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8844 - accuracy: 0.7674 - val_loss: 1.3263 - val_accuracy: 0.7046 Epoch 47/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8753 - accuracy: 0.7707 - val_loss: 1.3123 - val_accuracy: 0.7032 Epoch 48/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8737 - accuracy: 0.7691 - val_loss: 1.3136 - val_accuracy: 0.7042 Epoch 49/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8645 - accuracy: 0.7726 - val_loss: 1.3274 - val_accuracy: 0.7052 Epoch 50/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8709 - accuracy: 0.7702 - val_loss: 1.3239 - val_accuracy: 0.7063 Epoch 51/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8693 - accuracy: 0.7720 - val_loss: 1.3203 - val_accuracy: 0.7069 Epoch 52/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8572 - accuracy: 0.7733 - val_loss: 1.3149 - val_accuracy: 0.7071 Epoch 53/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8625 - accuracy: 0.7730 - val_loss: 1.3172 - val_accuracy: 0.7062 Epoch 54/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8591 - accuracy: 0.7734 - val_loss: 1.3204 - val_accuracy: 0.7037 Epoch 55/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8521 - accuracy: 0.7733 - val_loss: 1.3302 - val_accuracy: 0.7055 Epoch 56/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8514 - accuracy: 0.7740 - val_loss: 1.3228 - val_accuracy: 0.7051 Epoch 57/100 1148/1148 [==============================] - 9s 8ms/step - loss: 0.8470 - accuracy: 0.7745 - val_loss: 1.3205 - val_accuracy: 0.7051 Epoch 58/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8457 - accuracy: 0.7747 - val_loss: 1.3207 - val_accuracy: 0.7055 Epoch 59/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8445 - accuracy: 0.7772 - val_loss: 1.3280 - val_accuracy: 0.7074 Epoch 60/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8417 - accuracy: 0.7751 - val_loss: 1.3268 - val_accuracy: 0.7074 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8334 - accuracy: 0.7777 - val_loss: 1.3234 - val_accuracy: 0.7068 Epoch 62/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8372 - accuracy: 0.7777 - val_loss: 1.3243 - val_accuracy: 0.7093 Epoch 63/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8333 - accuracy: 0.7769 - val_loss: 1.3323 - val_accuracy: 0.7079 Epoch 64/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8315 - accuracy: 0.7791 - val_loss: 1.3203 - val_accuracy: 0.7061 Epoch 65/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8293 - accuracy: 0.7793 - val_loss: 1.3384 - val_accuracy: 0.7090 Epoch 66/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8215 - accuracy: 0.7782 - val_loss: 1.3460 - val_accuracy: 0.7074 Epoch 67/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8207 - accuracy: 0.7815 - val_loss: 1.3427 - val_accuracy: 0.7033 Epoch 68/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8168 - accuracy: 0.7817 - val_loss: 1.3369 - val_accuracy: 0.7069 Epoch 69/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8173 - accuracy: 0.7798 - val_loss: 1.3486 - val_accuracy: 0.7064 Epoch 70/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8193 - accuracy: 0.7810 - val_loss: 1.3394 - val_accuracy: 0.7070 Epoch 71/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8192 - accuracy: 0.7804 - val_loss: 1.3474 - val_accuracy: 0.7044 Epoch 72/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8143 - accuracy: 0.7810 - val_loss: 1.3443 - val_accuracy: 0.7077 Epoch 73/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8118 - accuracy: 0.7832 - val_loss: 1.3524 - val_accuracy: 0.7064 Epoch 74/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8149 - accuracy: 0.7813 - val_loss: 1.3446 - val_accuracy: 0.7052 Epoch 75/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8146 - accuracy: 0.7800 - val_loss: 1.3346 - val_accuracy: 0.7056 Epoch 76/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8047 - accuracy: 0.7824 - val_loss: 1.3397 - val_accuracy: 0.7096 Epoch 77/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8042 - accuracy: 0.7832 - val_loss: 1.3549 - val_accuracy: 0.7056 Epoch 78/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8124 - accuracy: 0.7805 - val_loss: 1.3439 - val_accuracy: 0.7080 Epoch 79/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8025 - accuracy: 0.7839 - val_loss: 1.3375 - val_accuracy: 0.7076 Epoch 80/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7999 - accuracy: 0.7840 - val_loss: 1.3513 - val_accuracy: 0.7049 Epoch 81/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8029 - accuracy: 0.7821 - val_loss: 1.3565 - val_accuracy: 0.7075 Epoch 82/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7997 - accuracy: 0.7844 - val_loss: 1.3439 - val_accuracy: 0.7082 Epoch 83/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.8033 - accuracy: 0.7832 - val_loss: 1.3475 - val_accuracy: 0.7086 Epoch 84/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7960 - accuracy: 0.7824 - val_loss: 1.3530 - val_accuracy: 0.7082 Epoch 85/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7943 - accuracy: 0.7837 - val_loss: 1.3620 - val_accuracy: 0.7048 Epoch 86/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7921 - accuracy: 0.7850 - val_loss: 1.3488 - val_accuracy: 0.7042 Epoch 87/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7957 - accuracy: 0.7837 - val_loss: 1.3554 - val_accuracy: 0.7079 Epoch 88/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7952 - accuracy: 0.7846 - val_loss: 1.3591 - val_accuracy: 0.7071 Epoch 89/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7901 - accuracy: 0.7846 - val_loss: 1.3516 - val_accuracy: 0.7069 Epoch 90/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7909 - accuracy: 0.7842 - val_loss: 1.3569 - val_accuracy: 0.7081 Epoch 91/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7905 - accuracy: 0.7860 - val_loss: 1.3466 - val_accuracy: 0.7077 Epoch 92/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7842 - accuracy: 0.7867 - val_loss: 1.3653 - val_accuracy: 0.7066 Epoch 93/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7845 - accuracy: 0.7858 - val_loss: 1.3679 - val_accuracy: 0.7081 Epoch 94/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7890 - accuracy: 0.7857 - val_loss: 1.3648 - val_accuracy: 0.7058 Epoch 95/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7839 - accuracy: 0.7868 - val_loss: 1.3639 - val_accuracy: 0.7063 Epoch 96/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7898 - accuracy: 0.7834 - val_loss: 1.3658 - val_accuracy: 0.7104 Epoch 97/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7831 - accuracy: 0.7863 - val_loss: 1.3779 - val_accuracy: 0.7081 Epoch 98/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7808 - accuracy: 0.7858 - val_loss: 1.3673 - val_accuracy: 0.7034 Epoch 99/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.7799 - accuracy: 0.7865 - val_loss: 1.3721 - val_accuracy: 0.7087 Epoch 100/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.7866 - accuracy: 0.7864 - val_loss: 1.3521 - val_accuracy: 0.7073
Bi_GRU_V2.summary()
Model: "bi_directional_gru_v2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 256) 107520
l)
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 427,653
Trainable params: 427,653
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_GRU_V2_history.history)
Observations
Bi_GRU_V2.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.3552 - accuracy: 0.7052
[1.3552422523498535, 0.7052141427993774]
Observations
tf.keras.backend.clear_session()
# Create the model
Bi_GRU_V3 = Sequential(
name='bi_directional_gru_v3',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
Bidirectional(GRU(64, activation='tanh')),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
Bi_GRU_V3.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
Bi_GRU_V3_history = Bi_GRU_V3.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 10s 7ms/step - loss: 5.0111 - accuracy: 0.1341 - val_loss: 4.3091 - val_accuracy: 0.1993 Epoch 2/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.9752 - accuracy: 0.2404 - val_loss: 3.6872 - val_accuracy: 0.2828 Epoch 3/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.4695 - accuracy: 0.3056 - val_loss: 3.2907 - val_accuracy: 0.3401 Epoch 4/100 1148/1148 [==============================] - 8s 7ms/step - loss: 3.1317 - accuracy: 0.3509 - val_loss: 3.0031 - val_accuracy: 0.3831 Epoch 5/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.8664 - accuracy: 0.3903 - val_loss: 2.7726 - val_accuracy: 0.4142 Epoch 6/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.6523 - accuracy: 0.4236 - val_loss: 2.5870 - val_accuracy: 0.4495 Epoch 7/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.4736 - accuracy: 0.4555 - val_loss: 2.4332 - val_accuracy: 0.4794 Epoch 8/100 1148/1148 [==============================] - 7s 6ms/step - loss: 2.3217 - accuracy: 0.4813 - val_loss: 2.2888 - val_accuracy: 0.5087 Epoch 9/100 1148/1148 [==============================] - 7s 6ms/step - loss: 2.1908 - accuracy: 0.5041 - val_loss: 2.1753 - val_accuracy: 0.5278 Epoch 10/100 1148/1148 [==============================] - 8s 7ms/step - loss: 2.0889 - accuracy: 0.5232 - val_loss: 2.0770 - val_accuracy: 0.5458 Epoch 11/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9928 - accuracy: 0.5372 - val_loss: 1.9916 - val_accuracy: 0.5630 Epoch 12/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.9023 - accuracy: 0.5557 - val_loss: 1.9103 - val_accuracy: 0.5798 Epoch 13/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.8409 - accuracy: 0.5674 - val_loss: 1.8532 - val_accuracy: 0.5897 Epoch 14/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7667 - accuracy: 0.5834 - val_loss: 1.7929 - val_accuracy: 0.6007 Epoch 15/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.7067 - accuracy: 0.5933 - val_loss: 1.7514 - val_accuracy: 0.6075 Epoch 16/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6576 - accuracy: 0.6056 - val_loss: 1.7053 - val_accuracy: 0.6195 Epoch 17/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.6242 - accuracy: 0.6109 - val_loss: 1.6779 - val_accuracy: 0.6232 Epoch 18/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5731 - accuracy: 0.6231 - val_loss: 1.6417 - val_accuracy: 0.6297 Epoch 19/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5307 - accuracy: 0.6288 - val_loss: 1.6184 - val_accuracy: 0.6309 Epoch 20/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.5036 - accuracy: 0.6348 - val_loss: 1.5937 - val_accuracy: 0.6412 Epoch 21/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.4697 - accuracy: 0.6390 - val_loss: 1.5686 - val_accuracy: 0.6456 Epoch 22/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.4433 - accuracy: 0.6468 - val_loss: 1.5462 - val_accuracy: 0.6460 Epoch 23/100 1148/1148 [==============================] - 7s 7ms/step - loss: 1.4253 - accuracy: 0.6545 - val_loss: 1.5288 - val_accuracy: 0.6523 Epoch 24/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3953 - accuracy: 0.6599 - val_loss: 1.5226 - val_accuracy: 0.6514 Epoch 25/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3715 - accuracy: 0.6638 - val_loss: 1.4992 - val_accuracy: 0.6580 Epoch 26/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3544 - accuracy: 0.6653 - val_loss: 1.4929 - val_accuracy: 0.6590 Epoch 27/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3318 - accuracy: 0.6696 - val_loss: 1.4683 - val_accuracy: 0.6616 Epoch 28/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.3180 - accuracy: 0.6732 - val_loss: 1.4728 - val_accuracy: 0.6655 Epoch 29/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2985 - accuracy: 0.6782 - val_loss: 1.4493 - val_accuracy: 0.6684 Epoch 30/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2895 - accuracy: 0.6800 - val_loss: 1.4427 - val_accuracy: 0.6679 Epoch 31/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2682 - accuracy: 0.6850 - val_loss: 1.4255 - val_accuracy: 0.6754 Epoch 32/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2519 - accuracy: 0.6868 - val_loss: 1.4179 - val_accuracy: 0.6751 Epoch 33/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2411 - accuracy: 0.6910 - val_loss: 1.4189 - val_accuracy: 0.6748 Epoch 34/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2296 - accuracy: 0.6956 - val_loss: 1.4130 - val_accuracy: 0.6764 Epoch 35/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.2232 - accuracy: 0.6936 - val_loss: 1.4040 - val_accuracy: 0.6786 Epoch 36/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2126 - accuracy: 0.6968 - val_loss: 1.3992 - val_accuracy: 0.6804 Epoch 37/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1971 - accuracy: 0.6998 - val_loss: 1.4069 - val_accuracy: 0.6769 Epoch 38/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1858 - accuracy: 0.7005 - val_loss: 1.3827 - val_accuracy: 0.6845 Epoch 39/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1756 - accuracy: 0.7016 - val_loss: 1.3807 - val_accuracy: 0.6852 Epoch 40/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1729 - accuracy: 0.7028 - val_loss: 1.3826 - val_accuracy: 0.6819 Epoch 41/100 1148/1148 [==============================] - 7s 7ms/step - loss: 1.1626 - accuracy: 0.7040 - val_loss: 1.3684 - val_accuracy: 0.6855 Epoch 42/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1447 - accuracy: 0.7122 - val_loss: 1.3625 - val_accuracy: 0.6884 Epoch 43/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1440 - accuracy: 0.7102 - val_loss: 1.3608 - val_accuracy: 0.6885 Epoch 44/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1383 - accuracy: 0.7117 - val_loss: 1.3594 - val_accuracy: 0.6862 Epoch 45/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1293 - accuracy: 0.7146 - val_loss: 1.3512 - val_accuracy: 0.6882 Epoch 46/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1243 - accuracy: 0.7159 - val_loss: 1.3528 - val_accuracy: 0.6877 Epoch 47/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1151 - accuracy: 0.7172 - val_loss: 1.3486 - val_accuracy: 0.6885 Epoch 48/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.1050 - accuracy: 0.7201 - val_loss: 1.3463 - val_accuracy: 0.6899 Epoch 49/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0912 - accuracy: 0.7235 - val_loss: 1.3430 - val_accuracy: 0.6917 Epoch 50/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0983 - accuracy: 0.7219 - val_loss: 1.3396 - val_accuracy: 0.6891 Epoch 51/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0929 - accuracy: 0.7255 - val_loss: 1.3429 - val_accuracy: 0.6910 Epoch 52/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0840 - accuracy: 0.7233 - val_loss: 1.3451 - val_accuracy: 0.6902 Epoch 53/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0769 - accuracy: 0.7275 - val_loss: 1.3302 - val_accuracy: 0.6923 Epoch 54/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0728 - accuracy: 0.7290 - val_loss: 1.3396 - val_accuracy: 0.6927 Epoch 55/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0658 - accuracy: 0.7293 - val_loss: 1.3271 - val_accuracy: 0.6939 Epoch 56/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0618 - accuracy: 0.7281 - val_loss: 1.3354 - val_accuracy: 0.6925 Epoch 57/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0584 - accuracy: 0.7296 - val_loss: 1.3342 - val_accuracy: 0.6952 Epoch 58/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0574 - accuracy: 0.7301 - val_loss: 1.3309 - val_accuracy: 0.6943 Epoch 59/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0431 - accuracy: 0.7319 - val_loss: 1.3315 - val_accuracy: 0.6953 Epoch 60/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0399 - accuracy: 0.7342 - val_loss: 1.3292 - val_accuracy: 0.6956 Epoch 61/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0329 - accuracy: 0.7360 - val_loss: 1.3293 - val_accuracy: 0.6956 Epoch 62/100 1148/1148 [==============================] - 7s 7ms/step - loss: 1.0286 - accuracy: 0.7379 - val_loss: 1.3274 - val_accuracy: 0.6977 Epoch 63/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0362 - accuracy: 0.7359 - val_loss: 1.3146 - val_accuracy: 0.6935 Epoch 64/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0250 - accuracy: 0.7371 - val_loss: 1.3321 - val_accuracy: 0.6952 Epoch 65/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0216 - accuracy: 0.7395 - val_loss: 1.3236 - val_accuracy: 0.6955 Epoch 66/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0204 - accuracy: 0.7387 - val_loss: 1.3286 - val_accuracy: 0.6929 Epoch 67/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0203 - accuracy: 0.7387 - val_loss: 1.3232 - val_accuracy: 0.6951 Epoch 68/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0116 - accuracy: 0.7391 - val_loss: 1.3172 - val_accuracy: 0.7006 Epoch 69/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0088 - accuracy: 0.7402 - val_loss: 1.3281 - val_accuracy: 0.6977 Epoch 70/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0075 - accuracy: 0.7412 - val_loss: 1.3204 - val_accuracy: 0.6968 Epoch 71/100 1148/1148 [==============================] - 8s 7ms/step - loss: 1.0100 - accuracy: 0.7395 - val_loss: 1.3190 - val_accuracy: 0.6979 Epoch 72/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9943 - accuracy: 0.7455 - val_loss: 1.3218 - val_accuracy: 0.6968 Epoch 73/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9935 - accuracy: 0.7452 - val_loss: 1.3141 - val_accuracy: 0.6991 Epoch 74/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9916 - accuracy: 0.7446 - val_loss: 1.3113 - val_accuracy: 0.6993 Epoch 75/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9854 - accuracy: 0.7454 - val_loss: 1.3157 - val_accuracy: 0.6965 Epoch 76/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9822 - accuracy: 0.7491 - val_loss: 1.3143 - val_accuracy: 0.7011 Epoch 77/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9897 - accuracy: 0.7457 - val_loss: 1.3058 - val_accuracy: 0.6989 Epoch 78/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9807 - accuracy: 0.7480 - val_loss: 1.3171 - val_accuracy: 0.6986 Epoch 79/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9789 - accuracy: 0.7468 - val_loss: 1.3211 - val_accuracy: 0.6989 Epoch 80/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9791 - accuracy: 0.7494 - val_loss: 1.3252 - val_accuracy: 0.6970 Epoch 81/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9697 - accuracy: 0.7504 - val_loss: 1.3232 - val_accuracy: 0.7001 Epoch 82/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9747 - accuracy: 0.7477 - val_loss: 1.3115 - val_accuracy: 0.7001 Epoch 83/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.9675 - accuracy: 0.7499 - val_loss: 1.3220 - val_accuracy: 0.7006 Epoch 84/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9683 - accuracy: 0.7511 - val_loss: 1.3117 - val_accuracy: 0.7010 Epoch 85/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9630 - accuracy: 0.7495 - val_loss: 1.3154 - val_accuracy: 0.7010 Epoch 86/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9652 - accuracy: 0.7498 - val_loss: 1.3078 - val_accuracy: 0.7017 Epoch 87/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9577 - accuracy: 0.7515 - val_loss: 1.3119 - val_accuracy: 0.7016 Epoch 88/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9544 - accuracy: 0.7548 - val_loss: 1.3118 - val_accuracy: 0.7004 Epoch 89/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9578 - accuracy: 0.7527 - val_loss: 1.3242 - val_accuracy: 0.7024 Epoch 90/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9545 - accuracy: 0.7517 - val_loss: 1.3130 - val_accuracy: 0.7024 Epoch 91/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.9502 - accuracy: 0.7527 - val_loss: 1.3147 - val_accuracy: 0.7001 Epoch 92/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9530 - accuracy: 0.7532 - val_loss: 1.3095 - val_accuracy: 0.7000 Epoch 93/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9575 - accuracy: 0.7519 - val_loss: 1.3076 - val_accuracy: 0.6999 Epoch 94/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.9438 - accuracy: 0.7547 - val_loss: 1.3199 - val_accuracy: 0.6998 Epoch 95/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9420 - accuracy: 0.7536 - val_loss: 1.3156 - val_accuracy: 0.7001 Epoch 96/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.9391 - accuracy: 0.7565 - val_loss: 1.3115 - val_accuracy: 0.7015 Epoch 97/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9410 - accuracy: 0.7542 - val_loss: 1.3166 - val_accuracy: 0.6997 Epoch 98/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9390 - accuracy: 0.7556 - val_loss: 1.3115 - val_accuracy: 0.7016 Epoch 99/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9358 - accuracy: 0.7570 - val_loss: 1.3165 - val_accuracy: 0.6981 Epoch 100/100 1148/1148 [==============================] - 8s 7ms/step - loss: 0.9296 - accuracy: 0.7577 - val_loss: 1.3036 - val_accuracy: 0.7029
Bi_GRU_V3.summary()
Model: "bi_directional_gru_v3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
bidirectional (Bidirectiona (None, 128) 29184
l)
dropout (Dropout) (None, 128) 0
dense (Dense) (None, 1199) 154671
=================================================================
Total params: 195,845
Trainable params: 195,845
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(Bi_GRU_V3_history.history)
Observations
Bi_GRU_V3.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.3052 - accuracy: 0.7028
[1.3051730394363403, 0.7027623653411865]
Observations
# Phrases used to start the prediction
seed_texts = ['embrace each day',
'radiate some',
'believe that',
"life's actual purpose is",
'dance through each and every',
'let your time and energy',
'every person is',
'our country Singapore is',
'planet earth is',
'morning and evening would make it']
Corpus BLEU (Bilingual Evaluation Understudy) Score:
Calculate Precision for each n-gram:
Calculate Brevity Penalty (BP):
Calculate BLEU Score:
BERTScore (Bidirectional Encoder Representations from Transformers)
Calculate Precision, Recall, and F1 for each token:
Calculate Weighted F1 for each token:
Calculate Overall BERTScore:
Perplexity
# Add in reference text
reference = []
# Headers include phrases that the seed texts have existing in the dataset
referenceHeader = [
"Embrace", "Radiate", "Believe in yourself", "Life's", "Dance through", "Let your",
"Every", "Singapore" ,"Our planet", "This morning, let"
]
for quote in data["Quotes"].values:
# Check if the quote in the dataset contains any of the mentioned headers
if any(text in quote for text in referenceHeader):
# Convert quote to lowercase except for Singapore
if "Singapore" not in quote:
quote = quote.lower()
# Remove any punctuations from the quotes
reference.append(re.sub('[,;.]','', quote).split(' '))
references =[reference]
# Evaluate the BLEU score
def evaluate_bleu_score(text):
candidate = ["".join([text]).split(" ")]
score = corpus_bleu(references, candidate, weights=(0.1, 0.2, 0.3, 0.4), smoothing_function=SmoothingFunction().method1)
print(f"BLEU Score: {score:.4f}")
return score
# Add in reference text
reference2 = []
# Headers include phrases that the seed texts have existing in the dataset
referenceHeader = [
"Embrace", "Radiate", "Believe in yourself", "Life's", "Dance through", "Let your",
"Every", "Singapore" ,"Our planet", "This morning, let"
]
for quote in data["Quotes"].values:
# Check if the quote in the dataset contains any of the mentioned headers
if any(text in quote for text in referenceHeader):
# Convert quote to lowercase except for Singapore
if "Singapore" not in quote:
quote = quote.lower()
# Remove any punctuations from the quotes
reference2.append(re.sub('[,;.]','', quote))
references2 =[reference2]
# Evaluate BERTScore
def evaluate_bert_score(text):
candidate = ["".join([text])]
# Calculate BERT score
precision, recall, f1 = score(candidate, references2, lang="en", verbose=False)
print(f"Precision: {precision.mean().item():.4f}")
print(f"Recall: {recall.mean().item():.4f}")
print(f"F1 Score: {f1.mean().item():.4f}")
# Return the mean of the scores
return precision.mean().item(), recall.mean().item(), f1.mean().item()
# Calculate perplexity
def calculate_perplexity(loss, num_words):
perplexity = 2**(-loss/num_words)
print(f"Perplexity: {perplexity:.4f}")
return perplexity
# Temperature is used to control the randomness of the prediction
# The higher the temperature, the more random the prediction
def predict_next_words(input_text, model, temperature, num_words=1):
log_likelihood = 0.0
for _ in range(num_words):
tokens = tokenizer.texts_to_sequences([input_text])[0]
tokens = pad_sequences([tokens], maxlen=max_sequence_rolling_len, padding='pre')
predicted_prob = model.predict(tokens, verbose=0)[0]
prediction = np.log(predicted_prob) / temperature
exp_preds = np.exp(prediction)
predicted_probs = exp_preds / np.sum(exp_preds)
chosen_word_index = np.random.choice(range(len(predicted_probs)), p=predicted_probs)
predicted_word = tokenizer.index_word[chosen_word_index]
input_text += " " + predicted_word
log_likelihood += np.log2(predicted_prob[chosen_word_index])
return input_text, log_likelihood
#Function to plot a graph to consolidate BLEU scores and perplexity
def calc_bleu_and_perplexity(model_name ,model, seed_texts, temperature, num_word=10):
bleu_scores = []
perplexity_scores = []
for seed_text in seed_texts:
prediction, loss = predict_next_words(seed_text, model, temperature, num_word)
print(''.join(prediction))
bleu_score = evaluate_bleu_score(prediction)
perplexity = calculate_perplexity(loss, len(prediction.split(' ')))
print()
bleu_scores.append(bleu_score)
perplexity_scores.append(perplexity)
# Plot a bar graph to show the BLEU scores
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10,8), sharex=True)
# Plot BLEU scores
ax1.bar(range(1, len(bleu_scores)+1), bleu_scores, color='blue', alpha=0.7)
ax1.set_title(f'BLEU Scores: {model_name}')
ax1.set_ylabel("BLEU Score", color="blue")
ax1.axhline(y=0.5, color='r', linestyle='-')
# Plot perplexity scores
ax2.bar(range(1, len(perplexity_scores)+1), perplexity_scores, color='green', alpha=0.7)
ax2.set_title(f'Perplexity Scores: {model_name}')
ax2.set_ylabel("Perplexity", color="green")
ax2.axhline(y=1.5, color='grey', linestyle='--')
plt.tight_layout()
plt.show()
calc_bleu_and_perplexity("simpleRNN", simpleRNN, seed_texts, 0.4)
embrace each day is a precious gift to true beauty of our planet BLEU Score: 0.1055 Perplexity: 1.8448 radiate some strength and let it be the compass that guides you BLEU Score: 0.7573 Perplexity: 1.5710 believe that powers your journey a tale worth telling with warmth and BLEU Score: 0.5820 Perplexity: 1.5738 life's actual purpose is a testament to the goodness in your soul filling it BLEU Score: 0.5965 Perplexity: 1.3119 dance through each and every will be a force that drives you forward is a BLEU Score: 0.4545 Perplexity: 1.7440 let your time and energy through the canvas of your journey a tale worth telling BLEU Score: 0.6125 Perplexity: 1.3063 every person is a celebration of our planet offers the promise of blossoming BLEU Score: 0.3883 Perplexity: 2.1174 our country Singapore is a testament to the beauty of our planet's breath is BLEU Score: 0.5723 Perplexity: 1.6722 planet earth is a wonder of the beauty of our uniqueness and the BLEU Score: 0.4135 Perplexity: 1.9249 morning and evening would make it be the compass that guides you home moments that define BLEU Score: 0.4516 Perplexity: 1.6176
Observations
calc_bleu_and_perplexity("simpleRNNv2", simpleRNNv2, seed_texts, 0.4)
embrace each day with a heart full of gratitude for it is the BLEU Score: 0.8735 Perplexity: 1.3612 radiate some authenticity for it is the essence of true beauty and BLEU Score: 0.7276 Perplexity: 1.8191 believe that propels you forward is a gift to the soul of BLEU Score: 0.7253 Perplexity: 1.1579 life's actual purpose is the heartbeat of a joyful heart welcome the opportunities that BLEU Score: 0.4329 Perplexity: 1.3372 dance through each and every moment is a chance for adventure is created by the BLEU Score: 0.5906 Perplexity: 1.5402 let your time and energy to change and hope and delectable that brightens someone's path BLEU Score: 0.1932 Perplexity: 2.0117 every person is the universe of experiences and emotions for they hold the BLEU Score: 0.5062 Perplexity: 1.7247 our country Singapore is reality and love that reside within you to your passions BLEU Score: 0.3354 Perplexity: 1.7224 planet earth is a sanctuary of serenity within you is a sanctuary of BLEU Score: 0.4212 Perplexity: 1.9181 morning and evening would make it is the foundation of a joyful heart welcome the opportunities BLEU Score: 0.4450 Perplexity: 1.1857
Observations
calc_bleu_and_perplexity("simpleRNNv3", simpleRNNv3, seed_texts, 0.4)
embrace each day is a canvas for new beginnings reminding us of the BLEU Score: 0.4824 Perplexity: 1.2750 radiate some grace and let it be the foundation of your strength BLEU Score: 0.8055 Perplexity: 1.5193 believe that propels you forward is a victory on the path to BLEU Score: 0.7802 Perplexity: 1.4699 life's actual purpose is the music of the soul a release from the chains BLEU Score: 0.7607 Perplexity: 1.3377 dance through each and every fresh opportunities to shine for it is the key to BLEU Score: 0.5100 Perplexity: 1.3587 let your time and energy of happiness and contentment reside within you moments of joy BLEU Score: 0.2598 Perplexity: 1.7443 every person is a symbol of your soul and growth and wisdom that BLEU Score: 0.2834 Perplexity: 2.1598 our country Singapore is a gateway to a brighter future in the realization of BLEU Score: 0.7607 Perplexity: 1.5304 planet earth is a step towards the tranquility of the heart a serene BLEU Score: 0.8303 Perplexity: 1.3725 morning and evening would make it with your journey a fresh start for new beginnings reminding BLEU Score: 0.0776 Perplexity: 1.4178
Observations
calc_bleu_and_perplexity("LSTM_V1", LSTM_V1, seed_texts, 0.4)
embrace each day with a heart full of gratitude and the seeds of BLEU Score: 0.6769 Perplexity: 1.4634 radiate some confidence and let it be the wind in your sails BLEU Score: 0.8055 Perplexity: 1.3113 believe that yourself for you have the potential to make a difference BLEU Score: 0.8125 Perplexity: 1.3783 life's actual purpose is the heartbeat of a joyful heart for it turns ordinary BLEU Score: 0.6544 Perplexity: 1.1565 dance through each and every we transform with life and the gentle goodbyes of this BLEU Score: 0.2985 Perplexity: 1.4207 let your time and energy are the moments that take our breath away the beauty BLEU Score: 0.4777 Perplexity: 1.3515 every person is a reflection of the beauty in our diversity and dreams BLEU Score: 0.6863 Perplexity: 1.0948 our country Singapore is a testament to the beauty of our uniqueness and let BLEU Score: 0.6156 Perplexity: 1.1281 planet earth is a tapestry of unity and joy will follow the world BLEU Score: 0.3915 Perplexity: 1.4722 morning and evening would make it to the treasure within your soul and let it sing BLEU Score: 0.4280 Perplexity: 1.1711
Observations
calc_bleu_and_perplexity("LSTM_V2", LSTM_V2, seed_texts, 0.2)
embrace each day is a precious gift a gift a reminder that you BLEU Score: 0.4223 Perplexity: 1.5902 radiate some enthusiasm and let it be the foundation of your strength BLEU Score: 0.8055 Perplexity: 1.7878 believe that heals and unites and resentment a path towards healing and BLEU Score: 0.6823 Perplexity: 1.5829 life's actual purpose is a testament to the beauty of our uniqueness and growth BLEU Score: 0.6156 Perplexity: 1.3850 dance through each and every opportunities to shine for they hold the keys to your BLEU Score: 0.5100 Perplexity: 1.2736 let your time and energy it is the music of a joyful heart for it BLEU Score: 0.5538 Perplexity: 1.4680 every person is a testament to the beauty of the human spirit a BLEU Score: 0.7744 Perplexity: 1.3263 our country Singapore is the heartbeat of our planet and let it be the BLEU Score: 0.5235 Perplexity: 1.6284 planet earth is a classroom where we learn to love and forgive and BLEU Score: 0.7440 Perplexity: 1.5427 morning and evening would make it can light up even the darkest days together wonder of BLEU Score: 0.4605 Perplexity: 1.3002
Observations
calc_bleu_and_perplexity("LSTM_V3", LSTM_V3, seed_texts, 0.4)
embrace each day with a dreams are the thresholds of life's transition from BLEU Score: 0.6381 Perplexity: 1.3360 radiate some resilience and let it be the cornerstone of your character BLEU Score: 0.8055 Perplexity: 1.6781 believe that every day is a gift of this morning hear the BLEU Score: 0.2541 Perplexity: 1.4962 life's actual purpose is a step towards miracles a path that success hope and BLEU Score: 0.2837 Perplexity: 1.7105 dance through each and every beautiful chaos park's charm singapore's nature is a treasure trove BLEU Score: 0.2943 Perplexity: 1.0954 let your time and energy and kindness and reality of the world of your heart BLEU Score: 0.0786 Perplexity: 1.3635 every person is a treasure trove the power of the heart a serene BLEU Score: 0.6381 Perplexity: 1.2642 our country Singapore is the music that fills the air with joy and lightheartedness BLEU Score: 0.7607 Perplexity: 1.1233 planet earth is a reminder of the beauty in our diversity and dreams BLEU Score: 0.6720 Perplexity: 1.5095 morning and evening would make it can light up the lives of the world transform around BLEU Score: 0.5249 Perplexity: 1.5046
Observations
calc_bleu_and_perplexity("LSTM_V4", LSTM_V4, seed_texts, 0.6)
embrace each day we take flight of kindness and joy that reside for BLEU Score: 0.0876 Perplexity: 2.0410 radiate some nature and let your heart be the conductor of your BLEU Score: 0.7106 Perplexity: 2.0889 believe that reverberates in the heart spreading warmth and happiness far and BLEU Score: 0.9068 Perplexity: 1.0578 life's actual purpose is a celebration of the journey we've traveled and inner peace BLEU Score: 0.5723 Perplexity: 1.6819 dance through each and every beautiful charm a fresh start and new beginnings reminding us BLEU Score: 0.0679 Perplexity: 1.5588 let your time and energy and joy and create a beacon of hope for our BLEU Score: 0.4031 Perplexity: 1.5650 every person is a reminder of the earth's life giving embrace this morning BLEU Score: 0.4095 Perplexity: 1.3336 our country Singapore is the foundation of your potential and resilience a life well BLEU Score: 0.3558 Perplexity: 1.8105 planet earth is the driving force behind a fulfilled life and aspirations and BLEU Score: 0.6092 Perplexity: 1.7412 morning and evening would make it you with moments of joy and happiness far and wide BLEU Score: 0.4101 Perplexity: 1.4037
Observations
calc_bleu_and_perplexity("GRU_V1", GRU_V1, seed_texts, 0.3)
embrace each day is a canvas for new beginnings reminding us of the BLEU Score: 0.4824 Perplexity: 1.0752 radiate some peace and let it be the sanctuary within your heart BLEU Score: 0.8055 Perplexity: 1.4308 believe that brightens the world around you go the canvas of your BLEU Score: 0.5959 Perplexity: 1.5195 life's actual purpose is the music of a joyful heart and soul and resilience BLEU Score: 0.5104 Perplexity: 1.5103 dance through each and every day is a precious gift a reminder of the marvel BLEU Score: 0.5189 Perplexity: 1.2529 let your time and energy it can light up even the darkest days and let BLEU Score: 0.5187 Perplexity: 1.4620 every person is the wisdom that resides within you free from the weight BLEU Score: 0.4702 Perplexity: 1.7480 our country Singapore is its legacy of love and compassion you leave behind your BLEU Score: 0.5465 Perplexity: 1.3640 planet earth is a reminder of the preciousness of life and joy in BLEU Score: 0.6381 Perplexity: 1.2973 morning and evening would make it than blessings to the world like the rising sun let BLEU Score: 0.3355 Perplexity: 1.5712
Observations
calc_bleu_and_perplexity("GRU_V2", GRU_V2, seed_texts, 0.1)
embrace each day with a heart full of gratitude for it is the BLEU Score: 0.8735 Perplexity: 1.1224 radiate some gratitude for it is the heartbeat of a joyful heart BLEU Score: 0.8055 Perplexity: 1.4275 believe that yourself and you will be a source of light for BLEU Score: 0.8125 Perplexity: 1.2205 life's actual purpose is the pursuit of our passions and dreams and aspirations and BLEU Score: 0.5845 Perplexity: 1.0996 dance through each and every inspire is a testament to the beauty of our uniqueness BLEU Score: 0.6125 Perplexity: 1.1297 let your time and energy will follow and become a beacon of light in the BLEU Score: 0.5114 Perplexity: 1.3926 every person is the jewels set in the crown of the sea of BLEU Score: 0.7281 Perplexity: 1.1199 our country Singapore is a testament to the nation's resilience and unwavering determination and BLEU Score: 0.6961 Perplexity: 1.1576 planet earth is a testament to your inner strength and resilience that resonates BLEU Score: 0.5714 Perplexity: 1.3073 morning and evening would make it is the compass of endless discovery soul and inner peace BLEU Score: 0.4312 Perplexity: 1.1988
Observations
calc_bleu_and_perplexity("GRU_V3", GRU_V3, seed_texts, 0.1)
embrace each day is a canvas for new beginnings reminding us of the BLEU Score: 0.4824 Perplexity: 1.1786 radiate some kindness and become a beacon of light in the world BLEU Score: 0.8055 Perplexity: 1.5946 believe that every day is a gift a reminder of the marvel BLEU Score: 0.7639 Perplexity: 1.3592 life's actual purpose is a liberation of the soul and heal and transform to BLEU Score: 0.5235 Perplexity: 1.5351 dance through each and every step towards the universe a reminder of the marvel the BLEU Score: 0.5906 Perplexity: 1.4934 let your time and energy of growth for they remind you of the brilliance of BLEU Score: 0.6243 Perplexity: 1.4270 every person is a canvas for new beginnings reminding us of the preciousness BLEU Score: 0.4824 Perplexity: 1.5081 our country Singapore is a testament to the beauty of our planet holds the BLEU Score: 0.5723 Perplexity: 1.5790 planet earth is a wonder for the canvas of your journey a tale BLEU Score: 0.4863 Perplexity: 1.6381 morning and evening would make it for the stars of coastal protection and stability will follow BLEU Score: 0.0654 Perplexity: 1.6303
Observations
calc_bleu_and_perplexity("GRU_V4", GRU_V4, seed_texts, 0.1)
embrace each day is a canvas for new beginnings and fresh beautiful self BLEU Score: 0.5836 Perplexity: 1.5638 radiate some grace and let it be the cornerstone of your character BLEU Score: 0.8055 Perplexity: 1.4449 believe that morning brings your way is a precious gift a gift BLEU Score: 0.0734 Perplexity: 1.7648 life's actual purpose is a classroom where we learn to love and let go BLEU Score: 0.7607 Perplexity: 1.1095 dance through each and every soul from the universe of experiences and soul a reminder BLEU Score: 0.2459 Perplexity: 1.7020 let your time and energy is a treasure in the chest of memories a testament BLEU Score: 0.5576 Perplexity: 1.2441 every person is a step towards a joyful heart is a canvas of BLEU Score: 0.5456 Perplexity: 1.4923 our country Singapore is the treasure chest of experience experience and experience and experience BLEU Score: 0.3543 Perplexity: 1.3887 planet earth is an investment in your future of your story and your BLEU Score: 0.5572 Perplexity: 1.2038 morning and evening would make it knows the path to true happiness and contentment reside within BLEU Score: 0.4612 Perplexity: 1.2081
Observations
calc_bleu_and_perplexity("Bi_LSTM_V1", Bi_LSTM_V1, seed_texts, 0.1)
embrace each day is a precious gift that protect our hearts and destined BLEU Score: 0.0744 Perplexity: 1.5317 radiate some peace and create a haven of serenity around you of BLEU Score: 0.7276 Perplexity: 1.3767 believe that every day is a precious gift of our dreams and BLEU Score: 0.3999 Perplexity: 1.5115 life's actual purpose is the pursuit of our heart's deepest desires is a canvas BLEU Score: 0.5723 Perplexity: 1.2799 dance through each and every moment into a gift that brightens someone's day leaving a BLEU Score: 0.6792 Perplexity: 1.2751 let your time and energy it can turn dreams into reality and becoming a vibrant BLEU Score: 0.4031 Perplexity: 1.2149 every person is the heart of a truly beautiful soul for they are BLEU Score: 0.3998 Perplexity: 1.2496 our country Singapore is a treasure trove in the moment of a well lived BLEU Score: 0.1872 Perplexity: 1.4106 planet earth is a reminder of the preciousness of life and let your BLEU Score: 0.7872 Perplexity: 1.2156 morning and evening would make it is a lullaby for the soul of our planet to BLEU Score: 0.5528 Perplexity: 1.3186
Observations
calc_bleu_and_perplexity("Bi_LSTM_V3", Bi_LSTM_V3, seed_texts, 0.3)
embrace each day with a heart full of gratitude for it will lift BLEU Score: 0.8735 Perplexity: 1.0704 radiate some compassion for it is the heartbeat of humanity we find BLEU Score: 0.6142 Perplexity: 1.5571 believe that brightens any obstacle that comes your way of your soul BLEU Score: 0.4911 Perplexity: 1.4967 life's actual purpose is the music of a heart at peace in our planet's BLEU Score: 0.5581 Perplexity: 1.2983 dance through each and every heart you to the horizon of our planet a testament BLEU Score: 0.1054 Perplexity: 1.4202 let your time and energy to chase your dreams and aspirations and world transform in BLEU Score: 0.0572 Perplexity: 1.1364 every person is the canvas of your destiny each decision shaping the masterpiece BLEU Score: 0.7744 Perplexity: 1.0852 our country Singapore is an opportunity for growth and learning for be the beacon BLEU Score: 0.4998 Perplexity: 1.3218 planet earth is a liberation of the soul from bitterness and lightheartedness in BLEU Score: 0.5965 Perplexity: 1.3632 morning and evening would make it brings your light of success and resilience of our planet BLEU Score: 0.2524 Perplexity: 1.6775
Observations
calc_bleu_and_perplexity("Bi_LSTM_V4", Bi_LSTM_V4, seed_texts, 0.3)
embrace each day with a heart full of gratitude and determination shape destinies BLEU Score: 0.7543 Perplexity: 1.6876 radiate some grace and let it be the fortress of your soul BLEU Score: 0.8055 Perplexity: 1.8912 believe that reverberates in the heart spreading warmth and happiness far and BLEU Score: 0.9068 Perplexity: 1.1312 life's actual purpose is the symphony of life and let your heart be the BLEU Score: 0.7607 Perplexity: 1.4338 dance through each and every step we take towards our dreams and aspirations and the BLEU Score: 0.5906 Perplexity: 1.5995 let your time and energy and let it be the foundation of your greatness of BLEU Score: 0.5722 Perplexity: 1.5539 every person is a gesture of hope for the future of our planet BLEU Score: 0.8463 Perplexity: 1.5053 our country Singapore is an investment in a brighter future in the realization of BLEU Score: 0.7607 Perplexity: 1.2228 planet earth is the light of experience they hold the promise of a BLEU Score: 0.3915 Perplexity: 1.5598 morning and evening would make it with intention and love that believe in our souls and BLEU Score: 0.2826 Perplexity: 1.4591
Observations
calc_bleu_and_perplexity("Bi_GRU_V1", Bi_GRU_V1, seed_texts, 0.1)
embrace each day with a heart full of gratitude and watch how it BLEU Score: 0.8735 Perplexity: 1.0841 radiate some peace and let it be the sanctuary within your heart BLEU Score: 0.8055 Perplexity: 1.2799 believe that yourself and you will find the way to success and BLEU Score: 0.7169 Perplexity: 1.4681 life's actual purpose is reflected in the eyes of those we love and forgive BLEU Score: 0.6544 Perplexity: 1.3148 dance through each and every heritage singapore's nature is a treasure trove the gentle goodbyes BLEU Score: 0.3256 Perplexity: 1.2817 let your time and energy to the world of life's refreshment and renewal a reminder BLEU Score: 0.3139 Perplexity: 1.4531 every person is its own song in your future of our planet and BLEU Score: 0.2775 Perplexity: 1.4622 our country Singapore is ruggedness for the is the heartbeat of a grateful heart BLEU Score: 0.4500 Perplexity: 1.6063 planet earth is a canvas for new beginnings reminding us of the preciousness BLEU Score: 0.4712 Perplexity: 1.2577 morning and evening would make it with the stories of our journey and growth for learning BLEU Score: 0.5043 Perplexity: 1.1424
Observations
calc_bleu_and_perplexity("Bi_GRU_V2", Bi_GRU_V2, seed_texts, 0.1)
embrace each day with a heart full of gratitude and joy that reverberates BLEU Score: 0.6769 Perplexity: 1.2586 radiate some authenticity for it is the truest expression of your being BLEU Score: 0.8055 Perplexity: 1.4279 believe that falls is a reminder of the preciousness of life and BLEU Score: 0.8597 Perplexity: 1.2371 life's actual purpose is a gift a reminder of the miracle of life and BLEU Score: 0.7211 Perplexity: 1.1249 dance through each and every shape your darkest nights shine for it is the key BLEU Score: 0.4258 Perplexity: 1.3764 let your time and energy new beginnings reminding us of the preciousness of every day BLEU Score: 0.2117 Perplexity: 1.1048 every person is your way to the world around you leave in every BLEU Score: 0.4075 Perplexity: 1.2706 our country Singapore is a testament to a nation's aspirations and spirit your heart BLEU Score: 0.4622 Perplexity: 1.4706 planet earth is the heartbeat of a life well lived life that vision BLEU Score: 0.3729 Perplexity: 1.4504 morning and evening would make it knows the path to true happiness and contentment resilient ignites BLEU Score: 0.4212 Perplexity: 1.1976
Observations
calc_bleu_and_perplexity("Bi_GRU_V3", Bi_GRU_V3, seed_texts, 0.1)
embrace each day with the greatest significance you can give to another to BLEU Score: 0.3915 Perplexity: 1.5861 radiate some confidence and let it be the foundation of your strength BLEU Score: 0.8055 Perplexity: 1.5591 believe that brightens someone's day leaving a trail of smiles and warmth BLEU Score: 0.9068 Perplexity: 1.1413 life's actual purpose is a testament to inner strength and resilience of our planet BLEU Score: 0.6780 Perplexity: 1.4356 dance through each and every opportunities to shine and heart is a gift a reminder BLEU Score: 0.3278 Perplexity: 1.6058 let your time and energy within them with life and you will light to even BLEU Score: 0.2229 Perplexity: 1.5320 every person is the compass of endless moments that propels you forward is BLEU Score: 0.4702 Perplexity: 1.3785 our country Singapore is a tapestry woven with threads of love and hope and BLEU Score: 0.6961 Perplexity: 1.1578 planet earth is a step towards witnessing miracles the extraordinary moments that defy BLEU Score: 0.7872 Perplexity: 1.2941 morning and evening would make it propels you towards your dreams and aspirations and new beginnings BLEU Score: 0.3639 Perplexity: 1.3311
Observations
class attention(Layer):
def __init__(self,**kwargs):
super(attention,self).__init__(**kwargs)
# Build method
def build(self,input_shape):
# Create a trainable weight variable for this layer
self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="he_normal")
# Create a trainable bias variable for this layer
self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")
super(attention, self).build(input_shape)
# Call method
def call(self,x):
# Computes the attention scores
et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
# Computes the attention weights using softmax activation function
at=K.softmax(et)
at=K.expand_dims(at,axis=-1)
# Compute the weighted sum of the input vectors
output=x*at
return K.sum(output,axis=1)
# Compute output shape
def compute_output_shape(self,input_shape):
return (input_shape[0],input_shape[-1])
# Returns a dictionary containing the configuration used to initialize this layer
def get_config(self):
return super(attention,self).get_config()
tf.keras.backend.clear_session()
# Create the model to add in the attention for the GRU model with 256 units
GRU_with_attention = Sequential(
name='GRU_with_attention',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(256, activation='tanh', return_sequences=True),
attention(),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_with_attention.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_with_attention_history = GRU_with_attention.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 16s 13ms/step - loss: 5.5021 - accuracy: 0.0758 - val_loss: 5.4139 - val_accuracy: 0.0748 Epoch 2/100 1148/1148 [==============================] - 7s 6ms/step - loss: 5.2011 - accuracy: 0.1002 - val_loss: 4.9225 - val_accuracy: 0.1339 Epoch 3/100 1148/1148 [==============================] - 7s 6ms/step - loss: 4.6384 - accuracy: 0.1534 - val_loss: 4.4547 - val_accuracy: 0.1709 Epoch 4/100 1148/1148 [==============================] - 7s 6ms/step - loss: 4.1699 - accuracy: 0.2063 - val_loss: 4.0326 - val_accuracy: 0.2332 Epoch 5/100 1148/1148 [==============================] - 7s 6ms/step - loss: 3.7454 - accuracy: 0.2693 - val_loss: 3.6740 - val_accuracy: 0.2907 Epoch 6/100 1148/1148 [==============================] - 6s 6ms/step - loss: 3.3814 - accuracy: 0.3247 - val_loss: 3.3617 - val_accuracy: 0.3374 Epoch 7/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0609 - accuracy: 0.3758 - val_loss: 3.0834 - val_accuracy: 0.3892 Epoch 8/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.7836 - accuracy: 0.4228 - val_loss: 2.8404 - val_accuracy: 0.4246 Epoch 9/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.5496 - accuracy: 0.4613 - val_loss: 2.6469 - val_accuracy: 0.4619 Epoch 10/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.3530 - accuracy: 0.4950 - val_loss: 2.4884 - val_accuracy: 0.4885 Epoch 11/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.1859 - accuracy: 0.5254 - val_loss: 2.3631 - val_accuracy: 0.5080 Epoch 12/100 1148/1148 [==============================] - 7s 6ms/step - loss: 2.0425 - accuracy: 0.5527 - val_loss: 2.2316 - val_accuracy: 0.5335 Epoch 13/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.9255 - accuracy: 0.5771 - val_loss: 2.1354 - val_accuracy: 0.5507 Epoch 14/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.8197 - accuracy: 0.5972 - val_loss: 2.0483 - val_accuracy: 0.5682 Epoch 15/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.7252 - accuracy: 0.6128 - val_loss: 1.9804 - val_accuracy: 0.5825 Epoch 16/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.6489 - accuracy: 0.6275 - val_loss: 1.9163 - val_accuracy: 0.5903 Epoch 17/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.5719 - accuracy: 0.6432 - val_loss: 1.8567 - val_accuracy: 0.6022 Epoch 18/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.5077 - accuracy: 0.6551 - val_loss: 1.8023 - val_accuracy: 0.6077 Epoch 19/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.4511 - accuracy: 0.6688 - val_loss: 1.7619 - val_accuracy: 0.6180 Epoch 20/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3988 - accuracy: 0.6766 - val_loss: 1.7279 - val_accuracy: 0.6235 Epoch 21/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3533 - accuracy: 0.6857 - val_loss: 1.6911 - val_accuracy: 0.6304 Epoch 22/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.3158 - accuracy: 0.6912 - val_loss: 1.6630 - val_accuracy: 0.6337 Epoch 23/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2786 - accuracy: 0.6997 - val_loss: 1.6315 - val_accuracy: 0.6417 Epoch 24/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2407 - accuracy: 0.7070 - val_loss: 1.6073 - val_accuracy: 0.6426 Epoch 25/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.2117 - accuracy: 0.7125 - val_loss: 1.5865 - val_accuracy: 0.6465 Epoch 26/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1857 - accuracy: 0.7188 - val_loss: 1.5645 - val_accuracy: 0.6465 Epoch 27/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1579 - accuracy: 0.7201 - val_loss: 1.5530 - val_accuracy: 0.6515 Epoch 28/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1364 - accuracy: 0.7261 - val_loss: 1.5373 - val_accuracy: 0.6517 Epoch 29/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1167 - accuracy: 0.7276 - val_loss: 1.5281 - val_accuracy: 0.6513 Epoch 30/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0965 - accuracy: 0.7313 - val_loss: 1.5134 - val_accuracy: 0.6565 Epoch 31/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0797 - accuracy: 0.7350 - val_loss: 1.5011 - val_accuracy: 0.6561 Epoch 32/100 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0637 - accuracy: 0.7388 - val_loss: 1.4941 - val_accuracy: 0.6618 Epoch 33/100 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0429 - accuracy: 0.7397 - val_loss: 1.4786 - val_accuracy: 0.6608 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0318 - accuracy: 0.7441 - val_loss: 1.4719 - val_accuracy: 0.6626 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0114 - accuracy: 0.7490 - val_loss: 1.4588 - val_accuracy: 0.6643 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0024 - accuracy: 0.7479 - val_loss: 1.4548 - val_accuracy: 0.6661 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9952 - accuracy: 0.7491 - val_loss: 1.4444 - val_accuracy: 0.6673 Epoch 38/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9807 - accuracy: 0.7507 - val_loss: 1.4442 - val_accuracy: 0.6693 Epoch 39/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9674 - accuracy: 0.7547 - val_loss: 1.4358 - val_accuracy: 0.6665 Epoch 40/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9582 - accuracy: 0.7571 - val_loss: 1.4303 - val_accuracy: 0.6712 Epoch 41/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9524 - accuracy: 0.7590 - val_loss: 1.4292 - val_accuracy: 0.6709 Epoch 42/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9453 - accuracy: 0.7571 - val_loss: 1.4330 - val_accuracy: 0.6696 Epoch 43/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9347 - accuracy: 0.7625 - val_loss: 1.4204 - val_accuracy: 0.6741 Epoch 44/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9267 - accuracy: 0.7621 - val_loss: 1.4262 - val_accuracy: 0.6702 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9211 - accuracy: 0.7657 - val_loss: 1.4166 - val_accuracy: 0.6695 Epoch 46/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9162 - accuracy: 0.7651 - val_loss: 1.4163 - val_accuracy: 0.6737 Epoch 47/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9143 - accuracy: 0.7631 - val_loss: 1.4178 - val_accuracy: 0.6727 Epoch 48/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9015 - accuracy: 0.7689 - val_loss: 1.4020 - val_accuracy: 0.6773 Epoch 49/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8994 - accuracy: 0.7680 - val_loss: 1.4074 - val_accuracy: 0.6748 Epoch 50/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8913 - accuracy: 0.7670 - val_loss: 1.4075 - val_accuracy: 0.6721 Epoch 51/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8873 - accuracy: 0.7694 - val_loss: 1.4122 - val_accuracy: 0.6741 Epoch 52/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8881 - accuracy: 0.7672 - val_loss: 1.4087 - val_accuracy: 0.6753 Epoch 53/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8776 - accuracy: 0.7697 - val_loss: 1.4088 - val_accuracy: 0.6766 Epoch 54/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8738 - accuracy: 0.7699 - val_loss: 1.4006 - val_accuracy: 0.6766 Epoch 55/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8707 - accuracy: 0.7723 - val_loss: 1.4068 - val_accuracy: 0.6733 Epoch 56/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8655 - accuracy: 0.7718 - val_loss: 1.3945 - val_accuracy: 0.6763 Epoch 57/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8634 - accuracy: 0.7728 - val_loss: 1.4014 - val_accuracy: 0.6774 Epoch 58/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8604 - accuracy: 0.7739 - val_loss: 1.3920 - val_accuracy: 0.6785 Epoch 59/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8556 - accuracy: 0.7739 - val_loss: 1.3963 - val_accuracy: 0.6797 Epoch 60/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8527 - accuracy: 0.7753 - val_loss: 1.3940 - val_accuracy: 0.6779 Epoch 61/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8476 - accuracy: 0.7757 - val_loss: 1.3988 - val_accuracy: 0.6799 Epoch 62/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8487 - accuracy: 0.7756 - val_loss: 1.3936 - val_accuracy: 0.6767 Epoch 63/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8476 - accuracy: 0.7759 - val_loss: 1.3930 - val_accuracy: 0.6776 Epoch 64/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8400 - accuracy: 0.7763 - val_loss: 1.3968 - val_accuracy: 0.6760 Epoch 65/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8385 - accuracy: 0.7763 - val_loss: 1.3985 - val_accuracy: 0.6789 Epoch 66/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8320 - accuracy: 0.7782 - val_loss: 1.3875 - val_accuracy: 0.6787 Epoch 67/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8356 - accuracy: 0.7769 - val_loss: 1.3859 - val_accuracy: 0.6770 Epoch 68/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8279 - accuracy: 0.7771 - val_loss: 1.3916 - val_accuracy: 0.6785 Epoch 69/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8307 - accuracy: 0.7797 - val_loss: 1.3871 - val_accuracy: 0.6809 Epoch 70/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8256 - accuracy: 0.7803 - val_loss: 1.3948 - val_accuracy: 0.6791 Epoch 71/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8240 - accuracy: 0.7776 - val_loss: 1.3932 - val_accuracy: 0.6814 Epoch 72/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8281 - accuracy: 0.7770 - val_loss: 1.3819 - val_accuracy: 0.6790 Epoch 73/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8202 - accuracy: 0.7802 - val_loss: 1.3939 - val_accuracy: 0.6799 Epoch 74/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8222 - accuracy: 0.7801 - val_loss: 1.3914 - val_accuracy: 0.6796 Epoch 75/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8201 - accuracy: 0.7790 - val_loss: 1.3960 - val_accuracy: 0.6784 Epoch 76/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8162 - accuracy: 0.7805 - val_loss: 1.3951 - val_accuracy: 0.6802 Epoch 77/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8119 - accuracy: 0.7811 - val_loss: 1.3919 - val_accuracy: 0.6814 Epoch 78/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8088 - accuracy: 0.7807 - val_loss: 1.3952 - val_accuracy: 0.6833 Epoch 79/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8096 - accuracy: 0.7830 - val_loss: 1.3978 - val_accuracy: 0.6827 Epoch 80/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8103 - accuracy: 0.7816 - val_loss: 1.3866 - val_accuracy: 0.6841 Epoch 81/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.8081 - accuracy: 0.7815 - val_loss: 1.3886 - val_accuracy: 0.6817 Epoch 82/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8060 - accuracy: 0.7831 - val_loss: 1.3945 - val_accuracy: 0.6814 Epoch 83/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8112 - accuracy: 0.7783 - val_loss: 1.3869 - val_accuracy: 0.6837 Epoch 84/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8072 - accuracy: 0.7806 - val_loss: 1.3924 - val_accuracy: 0.6847 Epoch 85/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7982 - accuracy: 0.7833 - val_loss: 1.3914 - val_accuracy: 0.6854 Epoch 86/100 1148/1148 [==============================] - 7s 7ms/step - loss: 0.8014 - accuracy: 0.7825 - val_loss: 1.4000 - val_accuracy: 0.6843 Epoch 87/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.8020 - accuracy: 0.7831 - val_loss: 1.3900 - val_accuracy: 0.6836 Epoch 88/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7976 - accuracy: 0.7841 - val_loss: 1.3905 - val_accuracy: 0.6844 Epoch 89/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7950 - accuracy: 0.7843 - val_loss: 1.3959 - val_accuracy: 0.6845 Epoch 90/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7969 - accuracy: 0.7830 - val_loss: 1.3898 - val_accuracy: 0.6829 Epoch 91/100 1148/1148 [==============================] - 6s 5ms/step - loss: 0.7964 - accuracy: 0.7822 - val_loss: 1.4025 - val_accuracy: 0.6809 Epoch 92/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7986 - accuracy: 0.7827 - val_loss: 1.4016 - val_accuracy: 0.6832 Epoch 93/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7898 - accuracy: 0.7851 - val_loss: 1.3995 - val_accuracy: 0.6839 Epoch 94/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7908 - accuracy: 0.7840 - val_loss: 1.3971 - val_accuracy: 0.6814 Epoch 95/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7887 - accuracy: 0.7863 - val_loss: 1.3960 - val_accuracy: 0.6849 Epoch 96/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7891 - accuracy: 0.7841 - val_loss: 1.4038 - val_accuracy: 0.6796 Epoch 97/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7924 - accuracy: 0.7845 - val_loss: 1.3944 - val_accuracy: 0.6833 Epoch 98/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7909 - accuracy: 0.7841 - val_loss: 1.3968 - val_accuracy: 0.6839 Epoch 99/100 1148/1148 [==============================] - 7s 6ms/step - loss: 0.7841 - accuracy: 0.7874 - val_loss: 1.4029 - val_accuracy: 0.6827 Epoch 100/100 1148/1148 [==============================] - 6s 6ms/step - loss: 0.7881 - accuracy: 0.7845 - val_loss: 1.3875 - val_accuracy: 0.6868
GRU_with_attention.summary()
Model: "GRU_with_attention"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 34, 256) 205824
attention (attention) (None, 256) 290
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 526,247
Trainable params: 526,247
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_with_attention_history.history)
Observations
GRU_with_attention.evaluate(X_test_roll, y_test_roll)
1/383 [..............................] - ETA: 6s - loss: 1.2117 - accuracy: 0.7500383/383 [==============================] - 1s 2ms/step - loss: 1.3926 - accuracy: 0.6821
[1.3926458358764648, 0.6820856332778931]
Observations
# Before
calc_bleu_and_perplexity("GRU_without_attention", GRU_V2, seed_texts, 0.1)
embrace each day with a heart full of gratitude for it is the BLEU Score: 0.8735 Perplexity: 1.1224 radiate some gratitude for it is the heartbeat of a joyful heart BLEU Score: 0.8055 Perplexity: 1.4275 believe that yourself and you will be a source of light for BLEU Score: 0.8125 Perplexity: 1.2205 life's actual purpose is the pursuit of our passions and dreams and aspirations and BLEU Score: 0.5845 Perplexity: 1.0996 dance through each and every inspire is a testament to the beauty of our uniqueness BLEU Score: 0.6125 Perplexity: 1.1297 let your time and energy will follow and unites will transform hearts and minds is BLEU Score: 0.1770 Perplexity: 1.7947 every person is the promise of becoming reality and let your heart be BLEU Score: 0.5679 Perplexity: 1.3729 our country Singapore is a testament to the nation's resilience and unwavering determination and BLEU Score: 0.6961 Perplexity: 1.1576 planet earth is a testament to your inner strength and resilience that resonates BLEU Score: 0.5714 Perplexity: 1.3073 morning and evening would make it is the driving force behind your actions be the embodiment BLEU Score: 0.6221 Perplexity: 1.3855
# After
calc_bleu_and_perplexity("GRU_with_attention", GRU_with_attention, seed_texts, 0.1)
embrace each day with a heart full of gratitude gratitude is a step BLEU Score: 0.7087 Perplexity: 1.3148 radiate some gratitude for it turns even the smallest gifts into treasures BLEU Score: 0.8055 Perplexity: 1.3092 believe that spreads are in the treasury of cherished memories a testament BLEU Score: 0.6363 Perplexity: 1.5293 life's actual purpose is the legacy of love and compassion our planet and aspirations BLEU Score: 0.3853 Perplexity: 1.5855 dance through each and every moment for a moment of a fresh start for new BLEU Score: 0.2213 Perplexity: 1.4826 let your time and energy with joy and kindness will light up the world creating BLEU Score: 0.3356 Perplexity: 1.7417 every person is a testament a gift to the boundless power of the BLEU Score: 0.6769 Perplexity: 1.5491 our country Singapore is a canvas of your destiny each decision shaping the masterpiece BLEU Score: 0.6544 Perplexity: 1.2263 planet earth is a treasure trove in the beauty in transitions in the BLEU Score: 0.5456 Perplexity: 1.4991 morning and evening would make it brings forth new beginnings to inspire change a new day BLEU Score: 0.1997 Perplexity: 1.4581
Observations
tf.keras.backend.clear_session()
# Create the model
LSTM_with_attention = Sequential(
name='LSTM_with_attention',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
LSTM(64, activation='tanh', return_sequences=True),
attention(),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
LSTM_with_attention.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
LSTM_with_attention_history = LSTM_with_attention.fit(
X_train_roll, y_train_roll,
epochs=100,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/100 1148/1148 [==============================] - 7s 6ms/step - loss: 5.5198 - accuracy: 0.0782 - val_loss: 5.4083 - val_accuracy: 0.0748 Epoch 2/100 1148/1148 [==============================] - 6s 5ms/step - loss: 5.2942 - accuracy: 0.0791 - val_loss: 5.2472 - val_accuracy: 0.0749 Epoch 3/100 1148/1148 [==============================] - 6s 5ms/step - loss: 5.1861 - accuracy: 0.0849 - val_loss: 5.1673 - val_accuracy: 0.0942 Epoch 4/100 1148/1148 [==============================] - 6s 5ms/step - loss: 5.1154 - accuracy: 0.0931 - val_loss: 5.0996 - val_accuracy: 0.0938 Epoch 5/100 1148/1148 [==============================] - 6s 5ms/step - loss: 5.0351 - accuracy: 0.0993 - val_loss: 5.0172 - val_accuracy: 0.1038 Epoch 6/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.9538 - accuracy: 0.1121 - val_loss: 4.9357 - val_accuracy: 0.1327 Epoch 7/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.8631 - accuracy: 0.1326 - val_loss: 4.8768 - val_accuracy: 0.1518 Epoch 8/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.7616 - accuracy: 0.1529 - val_loss: 4.7430 - val_accuracy: 0.1599 Epoch 9/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.6515 - accuracy: 0.1761 - val_loss: 4.6267 - val_accuracy: 0.1871 Epoch 10/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.5384 - accuracy: 0.1933 - val_loss: 4.5331 - val_accuracy: 0.1954 Epoch 11/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.4354 - accuracy: 0.2026 - val_loss: 4.4276 - val_accuracy: 0.2054 Epoch 12/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.3407 - accuracy: 0.2120 - val_loss: 4.3396 - val_accuracy: 0.2119 Epoch 13/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.2553 - accuracy: 0.2223 - val_loss: 4.2653 - val_accuracy: 0.2281 Epoch 14/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.1718 - accuracy: 0.2329 - val_loss: 4.1833 - val_accuracy: 0.2344 Epoch 15/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0961 - accuracy: 0.2449 - val_loss: 4.1158 - val_accuracy: 0.2529 Epoch 16/100 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0215 - accuracy: 0.2553 - val_loss: 4.0381 - val_accuracy: 0.2568 Epoch 17/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.9492 - accuracy: 0.2664 - val_loss: 3.9718 - val_accuracy: 0.2706 Epoch 18/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8824 - accuracy: 0.2741 - val_loss: 3.9112 - val_accuracy: 0.2801 Epoch 19/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8178 - accuracy: 0.2842 - val_loss: 3.8747 - val_accuracy: 0.2963 Epoch 20/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.7595 - accuracy: 0.2916 - val_loss: 3.7892 - val_accuracy: 0.3001 Epoch 21/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.7001 - accuracy: 0.2983 - val_loss: 3.7330 - val_accuracy: 0.3079 Epoch 22/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.6437 - accuracy: 0.3072 - val_loss: 3.6808 - val_accuracy: 0.3104 Epoch 23/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5915 - accuracy: 0.3127 - val_loss: 3.6319 - val_accuracy: 0.3147 Epoch 24/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5359 - accuracy: 0.3200 - val_loss: 3.5986 - val_accuracy: 0.3222 Epoch 25/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4885 - accuracy: 0.3269 - val_loss: 3.5352 - val_accuracy: 0.3321 Epoch 26/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4339 - accuracy: 0.3335 - val_loss: 3.4990 - val_accuracy: 0.3437 Epoch 27/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.3910 - accuracy: 0.3405 - val_loss: 3.4430 - val_accuracy: 0.3521 Epoch 28/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.3390 - accuracy: 0.3481 - val_loss: 3.4122 - val_accuracy: 0.3474 Epoch 29/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.2942 - accuracy: 0.3566 - val_loss: 3.3530 - val_accuracy: 0.3631 Epoch 30/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.2523 - accuracy: 0.3617 - val_loss: 3.3117 - val_accuracy: 0.3683 Epoch 31/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.2091 - accuracy: 0.3678 - val_loss: 3.2836 - val_accuracy: 0.3763 Epoch 32/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.1642 - accuracy: 0.3728 - val_loss: 3.2395 - val_accuracy: 0.3750 Epoch 33/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.1292 - accuracy: 0.3804 - val_loss: 3.1897 - val_accuracy: 0.3876 Epoch 34/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0899 - accuracy: 0.3835 - val_loss: 3.1583 - val_accuracy: 0.3916 Epoch 35/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0447 - accuracy: 0.3912 - val_loss: 3.1188 - val_accuracy: 0.3987 Epoch 36/100 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0129 - accuracy: 0.3960 - val_loss: 3.0897 - val_accuracy: 0.3968 Epoch 37/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9741 - accuracy: 0.3996 - val_loss: 3.0544 - val_accuracy: 0.4019 Epoch 38/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9353 - accuracy: 0.4057 - val_loss: 3.0211 - val_accuracy: 0.4157 Epoch 39/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9029 - accuracy: 0.4121 - val_loss: 2.9929 - val_accuracy: 0.4121 Epoch 40/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.8714 - accuracy: 0.4167 - val_loss: 2.9582 - val_accuracy: 0.4188 Epoch 41/100 1148/1148 [==============================] - 6s 6ms/step - loss: 2.8353 - accuracy: 0.4203 - val_loss: 2.9163 - val_accuracy: 0.4269 Epoch 42/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7991 - accuracy: 0.4264 - val_loss: 2.8935 - val_accuracy: 0.4332 Epoch 43/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7734 - accuracy: 0.4320 - val_loss: 2.8532 - val_accuracy: 0.4381 Epoch 44/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7420 - accuracy: 0.4353 - val_loss: 2.8318 - val_accuracy: 0.4395 Epoch 45/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7041 - accuracy: 0.4413 - val_loss: 2.8109 - val_accuracy: 0.4422 Epoch 46/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6812 - accuracy: 0.4440 - val_loss: 2.7734 - val_accuracy: 0.4550 Epoch 47/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6546 - accuracy: 0.4540 - val_loss: 2.7455 - val_accuracy: 0.4555 Epoch 48/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6223 - accuracy: 0.4540 - val_loss: 2.7228 - val_accuracy: 0.4591 Epoch 49/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5975 - accuracy: 0.4566 - val_loss: 2.7330 - val_accuracy: 0.4475 Epoch 50/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5670 - accuracy: 0.4639 - val_loss: 2.6735 - val_accuracy: 0.4641 Epoch 51/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5496 - accuracy: 0.4651 - val_loss: 2.6469 - val_accuracy: 0.4725 Epoch 52/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5196 - accuracy: 0.4706 - val_loss: 2.6206 - val_accuracy: 0.4751 Epoch 53/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4931 - accuracy: 0.4762 - val_loss: 2.5976 - val_accuracy: 0.4773 Epoch 54/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4711 - accuracy: 0.4786 - val_loss: 2.5801 - val_accuracy: 0.4808 Epoch 55/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4517 - accuracy: 0.4818 - val_loss: 2.5552 - val_accuracy: 0.4857 Epoch 56/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4236 - accuracy: 0.4874 - val_loss: 2.5540 - val_accuracy: 0.4890 Epoch 57/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4069 - accuracy: 0.4886 - val_loss: 2.5269 - val_accuracy: 0.4871 Epoch 58/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3880 - accuracy: 0.4914 - val_loss: 2.4944 - val_accuracy: 0.4970 Epoch 59/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3613 - accuracy: 0.4970 - val_loss: 2.4915 - val_accuracy: 0.4918 Epoch 60/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3490 - accuracy: 0.5011 - val_loss: 2.4620 - val_accuracy: 0.4958 Epoch 61/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3252 - accuracy: 0.5008 - val_loss: 2.4449 - val_accuracy: 0.5028 Epoch 62/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3075 - accuracy: 0.5063 - val_loss: 2.4297 - val_accuracy: 0.5040 Epoch 63/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2888 - accuracy: 0.5066 - val_loss: 2.4075 - val_accuracy: 0.5061 Epoch 64/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2671 - accuracy: 0.5119 - val_loss: 2.3905 - val_accuracy: 0.5120 Epoch 65/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2501 - accuracy: 0.5163 - val_loss: 2.3839 - val_accuracy: 0.5096 Epoch 66/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2331 - accuracy: 0.5206 - val_loss: 2.3609 - val_accuracy: 0.5130 Epoch 67/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2132 - accuracy: 0.5187 - val_loss: 2.3429 - val_accuracy: 0.5156 Epoch 68/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2022 - accuracy: 0.5242 - val_loss: 2.3223 - val_accuracy: 0.5225 Epoch 69/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1836 - accuracy: 0.5257 - val_loss: 2.3096 - val_accuracy: 0.5249 Epoch 70/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1636 - accuracy: 0.5319 - val_loss: 2.2950 - val_accuracy: 0.5217 Epoch 71/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1568 - accuracy: 0.5312 - val_loss: 2.2851 - val_accuracy: 0.5248 Epoch 72/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1392 - accuracy: 0.5348 - val_loss: 2.2645 - val_accuracy: 0.5297 Epoch 73/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1268 - accuracy: 0.5371 - val_loss: 2.2581 - val_accuracy: 0.5315 Epoch 74/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1043 - accuracy: 0.5395 - val_loss: 2.2398 - val_accuracy: 0.5337 Epoch 75/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0970 - accuracy: 0.5414 - val_loss: 2.2305 - val_accuracy: 0.5338 Epoch 76/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0871 - accuracy: 0.5451 - val_loss: 2.2154 - val_accuracy: 0.5381 Epoch 77/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0630 - accuracy: 0.5484 - val_loss: 2.2040 - val_accuracy: 0.5415 Epoch 78/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0527 - accuracy: 0.5513 - val_loss: 2.1908 - val_accuracy: 0.5468 Epoch 79/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0408 - accuracy: 0.5517 - val_loss: 2.1798 - val_accuracy: 0.5437 Epoch 80/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0271 - accuracy: 0.5541 - val_loss: 2.1727 - val_accuracy: 0.5458 Epoch 81/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0210 - accuracy: 0.5566 - val_loss: 2.1600 - val_accuracy: 0.5458 Epoch 82/100 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0058 - accuracy: 0.5560 - val_loss: 2.1440 - val_accuracy: 0.5505 Epoch 83/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9898 - accuracy: 0.5632 - val_loss: 2.1371 - val_accuracy: 0.5529 Epoch 84/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9868 - accuracy: 0.5617 - val_loss: 2.1302 - val_accuracy: 0.5534 Epoch 85/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9739 - accuracy: 0.5623 - val_loss: 2.1242 - val_accuracy: 0.5501 Epoch 86/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9612 - accuracy: 0.5646 - val_loss: 2.1062 - val_accuracy: 0.5579 Epoch 87/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9589 - accuracy: 0.5660 - val_loss: 2.0936 - val_accuracy: 0.5607 Epoch 88/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9441 - accuracy: 0.5677 - val_loss: 2.0914 - val_accuracy: 0.5584 Epoch 89/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9318 - accuracy: 0.5701 - val_loss: 2.0762 - val_accuracy: 0.5627 Epoch 90/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9246 - accuracy: 0.5746 - val_loss: 2.0723 - val_accuracy: 0.5602 Epoch 91/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9137 - accuracy: 0.5750 - val_loss: 2.0605 - val_accuracy: 0.5636 Epoch 92/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9029 - accuracy: 0.5769 - val_loss: 2.0498 - val_accuracy: 0.5653 Epoch 93/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8909 - accuracy: 0.5783 - val_loss: 2.0446 - val_accuracy: 0.5664 Epoch 94/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8839 - accuracy: 0.5792 - val_loss: 2.0331 - val_accuracy: 0.5691 Epoch 95/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8744 - accuracy: 0.5850 - val_loss: 2.0284 - val_accuracy: 0.5700 Epoch 96/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8647 - accuracy: 0.5846 - val_loss: 2.0169 - val_accuracy: 0.5733 Epoch 97/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8542 - accuracy: 0.5862 - val_loss: 2.0103 - val_accuracy: 0.5739 Epoch 98/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8480 - accuracy: 0.5871 - val_loss: 2.0052 - val_accuracy: 0.5733 Epoch 99/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8364 - accuracy: 0.5888 - val_loss: 1.9909 - val_accuracy: 0.5750 Epoch 100/100 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8339 - accuracy: 0.5895 - val_loss: 1.9905 - val_accuracy: 0.5763
LSTM_with_attention.summary()
Model: "LSTM_with_attention"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
lstm (LSTM) (None, 34, 64) 19200
attention (attention) (None, 64) 98
dropout (Dropout) (None, 64) 0
dense (Dense) (None, 1199) 77935
=================================================================
Total params: 109,223
Trainable params: 109,223
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(LSTM_with_attention_history.history)
Observations
LSTM_with_attention.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.9579 - accuracy: 0.5811
[1.957946538925171, 0.5810722708702087]
Observations
calc_bleu_and_perplexity("LSTM_without_attention", LSTM_V2, seed_texts, 0.1)
embrace each day is a precious gift a gift that your dreams and BLEU Score: 0.1035 Perplexity: 1.6664 radiate some confidence and let it be the spark that ignites positivity BLEU Score: 0.8055 Perplexity: 1.7350 believe that brightens someone's day leaving a trail of smiles and warmth BLEU Score: 0.9068 Perplexity: 1.2685 life's actual purpose is a testament to the beauty of the human spirit a BLEU Score: 0.6961 Perplexity: 1.3686 dance through each and every opportunities to shine for they hold the keys to your BLEU Score: 0.5100 Perplexity: 1.2736 let your time and energy it is the heartbeat of humanity and let your heart BLEU Score: 0.4646 Perplexity: 1.5373 every person is a testament to the beauty of the human spirit a BLEU Score: 0.7744 Perplexity: 1.3263 our country Singapore is the heartbeat of our planet and let it be the BLEU Score: 0.5235 Perplexity: 1.6284 planet earth is a classroom where we learn to love and forgive and BLEU Score: 0.7440 Perplexity: 1.5427 morning and evening would make it can light up even the darkest days together wonder of BLEU Score: 0.4605 Perplexity: 1.3002
Observations
calc_bleu_and_perplexity("LSTM_with_attention", LSTM_with_attention, seed_texts, 0.1)
embrace each day is a gift in the world of life giving the BLEU Score: 0.0990 Perplexity: 2.3971 radiate some kindness and let it be the foundation of your soul BLEU Score: 0.6764 Perplexity: 2.1509 believe that ignites your soul it can light up the world of BLEU Score: 0.5028 Perplexity: 2.9669 life's actual purpose is a testament to the beauty in the human and joy BLEU Score: 0.4218 Perplexity: 2.7394 dance through each and every day is a step towards a brighter future in the BLEU Score: 0.6118 Perplexity: 1.9689 let your time and energy we take flight of your kindness be the beacon of BLEU Score: 0.3139 Perplexity: 2.1783 every person is the promise of our planet and dreams and aspirations and BLEU Score: 0.1035 Perplexity: 1.9504 our country Singapore is a liberation of the world of the heart a serene BLEU Score: 0.5346 Perplexity: 2.6846 planet earth is a testament to the beauty of our planet holds the BLEU Score: 0.6198 Perplexity: 2.1657 morning and evening would make it can light up even the gloomiest days your journey and BLEU Score: 0.4715 Perplexity: 1.6041
Observations
Adam
SGD (Stochastic Gradient Descent)
# SGD optimizer
tf.keras.backend.clear_session()
# Create the model
GRU_SGD = Sequential(
name='GRU_SGD',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(256, activation='tanh'),
Dropout(0.3),
Dense(total_words_rolling, activation='softmax')
]
)
opt = SGD(learning_rate=0.001, momentum = 0.9, nesterov= True)
GRU_SGD.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_SGD_history = GRU_SGD.fit(
X_train_roll, y_train_roll,
epochs=200,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/200 1148/1148 [==============================] - 15s 13ms/step - loss: 6.7513 - accuracy: 0.0791 - val_loss: 5.8850 - val_accuracy: 0.0749 Epoch 2/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.6069 - accuracy: 0.0775 - val_loss: 5.4710 - val_accuracy: 0.0749 Epoch 3/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4680 - accuracy: 0.0785 - val_loss: 5.4288 - val_accuracy: 0.0749 Epoch 4/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4422 - accuracy: 0.0767 - val_loss: 5.4166 - val_accuracy: 0.0749 Epoch 5/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4315 - accuracy: 0.0766 - val_loss: 5.4157 - val_accuracy: 0.0748 Epoch 6/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4235 - accuracy: 0.0768 - val_loss: 5.4112 - val_accuracy: 0.0748 Epoch 7/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4188 - accuracy: 0.0768 - val_loss: 5.4142 - val_accuracy: 0.0748 Epoch 8/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4131 - accuracy: 0.0778 - val_loss: 5.4111 - val_accuracy: 0.0748 Epoch 9/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4108 - accuracy: 0.0760 - val_loss: 5.4086 - val_accuracy: 0.0748 Epoch 10/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4117 - accuracy: 0.0765 - val_loss: 5.4088 - val_accuracy: 0.0749 Epoch 11/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4098 - accuracy: 0.0777 - val_loss: 5.4081 - val_accuracy: 0.0748 Epoch 12/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4064 - accuracy: 0.0774 - val_loss: 5.4076 - val_accuracy: 0.0748 Epoch 13/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4048 - accuracy: 0.0778 - val_loss: 5.4103 - val_accuracy: 0.0748 Epoch 14/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4035 - accuracy: 0.0768 - val_loss: 5.4078 - val_accuracy: 0.0748 Epoch 15/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4022 - accuracy: 0.0777 - val_loss: 5.4108 - val_accuracy: 0.0749 Epoch 16/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4021 - accuracy: 0.0773 - val_loss: 5.4063 - val_accuracy: 0.0865 Epoch 17/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4012 - accuracy: 0.0771 - val_loss: 5.4054 - val_accuracy: 0.0748 Epoch 18/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.4008 - accuracy: 0.0776 - val_loss: 5.4070 - val_accuracy: 0.0748 Epoch 19/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3983 - accuracy: 0.0793 - val_loss: 5.4146 - val_accuracy: 0.0748 Epoch 20/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3966 - accuracy: 0.0804 - val_loss: 5.4046 - val_accuracy: 0.0748 Epoch 21/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3967 - accuracy: 0.0790 - val_loss: 5.4055 - val_accuracy: 0.0748 Epoch 22/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3950 - accuracy: 0.0819 - val_loss: 5.4040 - val_accuracy: 0.0748 Epoch 23/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3957 - accuracy: 0.0813 - val_loss: 5.4050 - val_accuracy: 0.0748 Epoch 24/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3934 - accuracy: 0.0840 - val_loss: 5.4032 - val_accuracy: 0.0749 Epoch 25/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.3916 - accuracy: 0.0834 - val_loss: 5.4012 - val_accuracy: 0.0833 Epoch 26/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.3895 - accuracy: 0.0861 - val_loss: 5.3983 - val_accuracy: 0.1134 Epoch 27/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.3857 - accuracy: 0.0891 - val_loss: 5.3943 - val_accuracy: 0.1113 Epoch 28/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.3820 - accuracy: 0.0951 - val_loss: 5.3900 - val_accuracy: 0.0856 Epoch 29/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3735 - accuracy: 0.1008 - val_loss: 5.3814 - val_accuracy: 0.1250 Epoch 30/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3632 - accuracy: 0.1120 - val_loss: 5.3679 - val_accuracy: 0.1254 Epoch 31/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3449 - accuracy: 0.1201 - val_loss: 5.3476 - val_accuracy: 0.1194 Epoch 32/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.3165 - accuracy: 0.1255 - val_loss: 5.3130 - val_accuracy: 0.1254 Epoch 33/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.2803 - accuracy: 0.1286 - val_loss: 5.2683 - val_accuracy: 0.1259 Epoch 34/200 1148/1148 [==============================] - 6s 6ms/step - loss: 5.2150 - accuracy: 0.1310 - val_loss: 5.1930 - val_accuracy: 0.1247 Epoch 35/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.1368 - accuracy: 0.1331 - val_loss: 5.1213 - val_accuracy: 0.1275 Epoch 36/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.0712 - accuracy: 0.1383 - val_loss: 5.0635 - val_accuracy: 0.1420 Epoch 37/200 1148/1148 [==============================] - 6s 5ms/step - loss: 5.0223 - accuracy: 0.1471 - val_loss: 5.0192 - val_accuracy: 0.1528 Epoch 38/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.9815 - accuracy: 0.1542 - val_loss: 4.9815 - val_accuracy: 0.1497 Epoch 39/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.9437 - accuracy: 0.1601 - val_loss: 4.9446 - val_accuracy: 0.1533 Epoch 40/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.9115 - accuracy: 0.1621 - val_loss: 4.9126 - val_accuracy: 0.1576 Epoch 41/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.8768 - accuracy: 0.1631 - val_loss: 4.8816 - val_accuracy: 0.1561 Epoch 42/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.8456 - accuracy: 0.1670 - val_loss: 4.8517 - val_accuracy: 0.1547 Epoch 43/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.8104 - accuracy: 0.1677 - val_loss: 4.8188 - val_accuracy: 0.1680 Epoch 44/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.7758 - accuracy: 0.1723 - val_loss: 4.7842 - val_accuracy: 0.1704 Epoch 45/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.7421 - accuracy: 0.1732 - val_loss: 4.7480 - val_accuracy: 0.1720 Epoch 46/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.7028 - accuracy: 0.1762 - val_loss: 4.7072 - val_accuracy: 0.1743 Epoch 47/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.6597 - accuracy: 0.1816 - val_loss: 4.6672 - val_accuracy: 0.1791 Epoch 48/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.6173 - accuracy: 0.1882 - val_loss: 4.6238 - val_accuracy: 0.1866 Epoch 49/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.5736 - accuracy: 0.1927 - val_loss: 4.5796 - val_accuracy: 0.1911 Epoch 50/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.5279 - accuracy: 0.1971 - val_loss: 4.5323 - val_accuracy: 0.1936 Epoch 51/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.4789 - accuracy: 0.2032 - val_loss: 4.4853 - val_accuracy: 0.2001 Epoch 52/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.4311 - accuracy: 0.2079 - val_loss: 4.4379 - val_accuracy: 0.2072 Epoch 53/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.3834 - accuracy: 0.2144 - val_loss: 4.3878 - val_accuracy: 0.2104 Epoch 54/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.3320 - accuracy: 0.2191 - val_loss: 4.3409 - val_accuracy: 0.2262 Epoch 55/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.2851 - accuracy: 0.2260 - val_loss: 4.2929 - val_accuracy: 0.2248 Epoch 56/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.2353 - accuracy: 0.2331 - val_loss: 4.2499 - val_accuracy: 0.2251 Epoch 57/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.1886 - accuracy: 0.2362 - val_loss: 4.2000 - val_accuracy: 0.2406 Epoch 58/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.1447 - accuracy: 0.2423 - val_loss: 4.1573 - val_accuracy: 0.2418 Epoch 59/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0967 - accuracy: 0.2437 - val_loss: 4.1123 - val_accuracy: 0.2487 Epoch 60/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0539 - accuracy: 0.2482 - val_loss: 4.0701 - val_accuracy: 0.2490 Epoch 61/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0087 - accuracy: 0.2536 - val_loss: 4.0315 - val_accuracy: 0.2556 Epoch 62/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.9659 - accuracy: 0.2564 - val_loss: 3.9896 - val_accuracy: 0.2611 Epoch 63/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.9278 - accuracy: 0.2627 - val_loss: 3.9529 - val_accuracy: 0.2668 Epoch 64/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8885 - accuracy: 0.2654 - val_loss: 3.9157 - val_accuracy: 0.2735 Epoch 65/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8492 - accuracy: 0.2718 - val_loss: 3.8751 - val_accuracy: 0.2797 Epoch 66/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.8122 - accuracy: 0.2745 - val_loss: 3.8384 - val_accuracy: 0.2815 Epoch 67/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.7679 - accuracy: 0.2790 - val_loss: 3.8048 - val_accuracy: 0.2892 Epoch 68/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.7357 - accuracy: 0.2842 - val_loss: 3.7683 - val_accuracy: 0.2896 Epoch 69/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.6945 - accuracy: 0.2878 - val_loss: 3.7337 - val_accuracy: 0.2987 Epoch 70/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.6587 - accuracy: 0.2939 - val_loss: 3.6958 - val_accuracy: 0.3027 Epoch 71/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.6220 - accuracy: 0.2978 - val_loss: 3.6658 - val_accuracy: 0.3017 Epoch 72/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5894 - accuracy: 0.3015 - val_loss: 3.6296 - val_accuracy: 0.3112 Epoch 73/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5511 - accuracy: 0.3051 - val_loss: 3.5962 - val_accuracy: 0.3142 Epoch 74/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5179 - accuracy: 0.3118 - val_loss: 3.5654 - val_accuracy: 0.3195 Epoch 75/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4852 - accuracy: 0.3165 - val_loss: 3.5342 - val_accuracy: 0.3283 Epoch 76/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4493 - accuracy: 0.3195 - val_loss: 3.5050 - val_accuracy: 0.3301 Epoch 77/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.4204 - accuracy: 0.3234 - val_loss: 3.4731 - val_accuracy: 0.3322 Epoch 78/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.3864 - accuracy: 0.3274 - val_loss: 3.4495 - val_accuracy: 0.3352 Epoch 79/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.3549 - accuracy: 0.3312 - val_loss: 3.4186 - val_accuracy: 0.3380 Epoch 80/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.3261 - accuracy: 0.3355 - val_loss: 3.3894 - val_accuracy: 0.3397 Epoch 81/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.2942 - accuracy: 0.3393 - val_loss: 3.3635 - val_accuracy: 0.3472 Epoch 82/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.2687 - accuracy: 0.3429 - val_loss: 3.3361 - val_accuracy: 0.3496 Epoch 83/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.2346 - accuracy: 0.3472 - val_loss: 3.3130 - val_accuracy: 0.3511 Epoch 84/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.2125 - accuracy: 0.3488 - val_loss: 3.2838 - val_accuracy: 0.3554 Epoch 85/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.1801 - accuracy: 0.3562 - val_loss: 3.2591 - val_accuracy: 0.3586 Epoch 86/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.1546 - accuracy: 0.3596 - val_loss: 3.2300 - val_accuracy: 0.3620 Epoch 87/200 1148/1148 [==============================] - 7s 6ms/step - loss: 3.1289 - accuracy: 0.3615 - val_loss: 3.2054 - val_accuracy: 0.3662 Epoch 88/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.1012 - accuracy: 0.3659 - val_loss: 3.1824 - val_accuracy: 0.3701 Epoch 89/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0732 - accuracy: 0.3696 - val_loss: 3.1540 - val_accuracy: 0.3746 Epoch 90/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0469 - accuracy: 0.3735 - val_loss: 3.1357 - val_accuracy: 0.3752 Epoch 91/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.0225 - accuracy: 0.3767 - val_loss: 3.1097 - val_accuracy: 0.3824 Epoch 92/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9981 - accuracy: 0.3811 - val_loss: 3.0850 - val_accuracy: 0.3839 Epoch 93/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9687 - accuracy: 0.3836 - val_loss: 3.0665 - val_accuracy: 0.3877 Epoch 94/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9484 - accuracy: 0.3889 - val_loss: 3.0419 - val_accuracy: 0.3935 Epoch 95/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9167 - accuracy: 0.3916 - val_loss: 3.0213 - val_accuracy: 0.3954 Epoch 96/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.8971 - accuracy: 0.3945 - val_loss: 3.0010 - val_accuracy: 0.4027 Epoch 97/200 1148/1148 [==============================] - 6s 6ms/step - loss: 2.8712 - accuracy: 0.4004 - val_loss: 2.9765 - val_accuracy: 0.4072 Epoch 98/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.8493 - accuracy: 0.4035 - val_loss: 2.9591 - val_accuracy: 0.4069 Epoch 99/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.8252 - accuracy: 0.4068 - val_loss: 2.9374 - val_accuracy: 0.4103 Epoch 100/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.8044 - accuracy: 0.4079 - val_loss: 2.9102 - val_accuracy: 0.4153 Epoch 101/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7796 - accuracy: 0.4135 - val_loss: 2.8959 - val_accuracy: 0.4140 Epoch 102/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7593 - accuracy: 0.4181 - val_loss: 2.8721 - val_accuracy: 0.4203 Epoch 103/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7397 - accuracy: 0.4192 - val_loss: 2.8545 - val_accuracy: 0.4215 Epoch 104/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7197 - accuracy: 0.4239 - val_loss: 2.8331 - val_accuracy: 0.4276 Epoch 105/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6963 - accuracy: 0.4278 - val_loss: 2.8139 - val_accuracy: 0.4310 Epoch 106/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6737 - accuracy: 0.4308 - val_loss: 2.7952 - val_accuracy: 0.4307 Epoch 107/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6538 - accuracy: 0.4334 - val_loss: 2.7759 - val_accuracy: 0.4358 Epoch 108/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6379 - accuracy: 0.4348 - val_loss: 2.7590 - val_accuracy: 0.4376 Epoch 109/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.6159 - accuracy: 0.4403 - val_loss: 2.7435 - val_accuracy: 0.4389 Epoch 110/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5972 - accuracy: 0.4433 - val_loss: 2.7205 - val_accuracy: 0.4427 Epoch 111/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5713 - accuracy: 0.4485 - val_loss: 2.7050 - val_accuracy: 0.4434 Epoch 112/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5561 - accuracy: 0.4467 - val_loss: 2.6903 - val_accuracy: 0.4495 Epoch 113/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5364 - accuracy: 0.4511 - val_loss: 2.6712 - val_accuracy: 0.4530 Epoch 114/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5225 - accuracy: 0.4532 - val_loss: 2.6499 - val_accuracy: 0.4569 Epoch 115/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5047 - accuracy: 0.4576 - val_loss: 2.6333 - val_accuracy: 0.4598 Epoch 116/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4821 - accuracy: 0.4605 - val_loss: 2.6233 - val_accuracy: 0.4591 Epoch 117/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4685 - accuracy: 0.4641 - val_loss: 2.6065 - val_accuracy: 0.4630 Epoch 118/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4487 - accuracy: 0.4691 - val_loss: 2.5923 - val_accuracy: 0.4691 Epoch 119/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4321 - accuracy: 0.4710 - val_loss: 2.5715 - val_accuracy: 0.4729 Epoch 120/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4183 - accuracy: 0.4731 - val_loss: 2.5582 - val_accuracy: 0.4741 Epoch 121/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3990 - accuracy: 0.4781 - val_loss: 2.5440 - val_accuracy: 0.4752 Epoch 122/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3842 - accuracy: 0.4819 - val_loss: 2.5264 - val_accuracy: 0.4781 Epoch 123/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3657 - accuracy: 0.4814 - val_loss: 2.5139 - val_accuracy: 0.4772 Epoch 124/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3469 - accuracy: 0.4854 - val_loss: 2.4943 - val_accuracy: 0.4809 Epoch 125/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3379 - accuracy: 0.4855 - val_loss: 2.4812 - val_accuracy: 0.4864 Epoch 126/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3142 - accuracy: 0.4925 - val_loss: 2.4693 - val_accuracy: 0.4872 Epoch 127/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2961 - accuracy: 0.4916 - val_loss: 2.4551 - val_accuracy: 0.4911 Epoch 128/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2918 - accuracy: 0.4980 - val_loss: 2.4405 - val_accuracy: 0.4937 Epoch 129/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2755 - accuracy: 0.4991 - val_loss: 2.4307 - val_accuracy: 0.4931 Epoch 130/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2538 - accuracy: 0.5017 - val_loss: 2.4124 - val_accuracy: 0.4958 Epoch 131/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2435 - accuracy: 0.5031 - val_loss: 2.3993 - val_accuracy: 0.4963 Epoch 132/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2321 - accuracy: 0.5055 - val_loss: 2.3861 - val_accuracy: 0.5013 Epoch 133/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2165 - accuracy: 0.5080 - val_loss: 2.3776 - val_accuracy: 0.5062 Epoch 134/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1988 - accuracy: 0.5134 - val_loss: 2.3644 - val_accuracy: 0.5013 Epoch 135/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1871 - accuracy: 0.5133 - val_loss: 2.3481 - val_accuracy: 0.5080 Epoch 136/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1742 - accuracy: 0.5178 - val_loss: 2.3387 - val_accuracy: 0.5088 Epoch 137/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1614 - accuracy: 0.5194 - val_loss: 2.3264 - val_accuracy: 0.5093 Epoch 138/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1488 - accuracy: 0.5218 - val_loss: 2.3119 - val_accuracy: 0.5123 Epoch 139/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1348 - accuracy: 0.5246 - val_loss: 2.3024 - val_accuracy: 0.5128 Epoch 140/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1265 - accuracy: 0.5246 - val_loss: 2.2945 - val_accuracy: 0.5144 Epoch 141/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1104 - accuracy: 0.5275 - val_loss: 2.2783 - val_accuracy: 0.5184 Epoch 142/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0969 - accuracy: 0.5294 - val_loss: 2.2685 - val_accuracy: 0.5201 Epoch 143/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0847 - accuracy: 0.5341 - val_loss: 2.2595 - val_accuracy: 0.5207 Epoch 144/200 1148/1148 [==============================] - 7s 6ms/step - loss: 2.0786 - accuracy: 0.5347 - val_loss: 2.2453 - val_accuracy: 0.5270 Epoch 145/200 1148/1148 [==============================] - 7s 6ms/step - loss: 2.0633 - accuracy: 0.5364 - val_loss: 2.2333 - val_accuracy: 0.5254 Epoch 146/200 1148/1148 [==============================] - 6s 6ms/step - loss: 2.0556 - accuracy: 0.5369 - val_loss: 2.2241 - val_accuracy: 0.5285 Epoch 147/200 1148/1148 [==============================] - 7s 6ms/step - loss: 2.0439 - accuracy: 0.5411 - val_loss: 2.2157 - val_accuracy: 0.5297 Epoch 148/200 1148/1148 [==============================] - 6s 6ms/step - loss: 2.0302 - accuracy: 0.5421 - val_loss: 2.2035 - val_accuracy: 0.5316 Epoch 149/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0185 - accuracy: 0.5452 - val_loss: 2.1950 - val_accuracy: 0.5335 Epoch 150/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0102 - accuracy: 0.5461 - val_loss: 2.1868 - val_accuracy: 0.5351 Epoch 151/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9979 - accuracy: 0.5501 - val_loss: 2.1775 - val_accuracy: 0.5380 Epoch 152/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9856 - accuracy: 0.5493 - val_loss: 2.1636 - val_accuracy: 0.5387 Epoch 153/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9795 - accuracy: 0.5522 - val_loss: 2.1676 - val_accuracy: 0.5369 Epoch 154/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9691 - accuracy: 0.5541 - val_loss: 2.1492 - val_accuracy: 0.5408 Epoch 155/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9555 - accuracy: 0.5563 - val_loss: 2.1378 - val_accuracy: 0.5409 Epoch 156/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9470 - accuracy: 0.5562 - val_loss: 2.1273 - val_accuracy: 0.5474 Epoch 157/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9405 - accuracy: 0.5589 - val_loss: 2.1276 - val_accuracy: 0.5440 Epoch 158/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9290 - accuracy: 0.5613 - val_loss: 2.1152 - val_accuracy: 0.5446 Epoch 159/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.9246 - accuracy: 0.5601 - val_loss: 2.1034 - val_accuracy: 0.5517 Epoch 160/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.9070 - accuracy: 0.5661 - val_loss: 2.0937 - val_accuracy: 0.5516 Epoch 161/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8995 - accuracy: 0.5679 - val_loss: 2.0879 - val_accuracy: 0.5523 Epoch 162/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8913 - accuracy: 0.5690 - val_loss: 2.0800 - val_accuracy: 0.5523 Epoch 163/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8774 - accuracy: 0.5704 - val_loss: 2.0685 - val_accuracy: 0.5552 Epoch 164/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8717 - accuracy: 0.5740 - val_loss: 2.0677 - val_accuracy: 0.5567 Epoch 165/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8616 - accuracy: 0.5760 - val_loss: 2.0563 - val_accuracy: 0.5552 Epoch 166/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8549 - accuracy: 0.5731 - val_loss: 2.0451 - val_accuracy: 0.5609 Epoch 167/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8460 - accuracy: 0.5751 - val_loss: 2.0362 - val_accuracy: 0.5592 Epoch 168/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8333 - accuracy: 0.5775 - val_loss: 2.0287 - val_accuracy: 0.5620 Epoch 169/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8282 - accuracy: 0.5797 - val_loss: 2.0231 - val_accuracy: 0.5640 Epoch 170/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8207 - accuracy: 0.5814 - val_loss: 2.0176 - val_accuracy: 0.5666 Epoch 171/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8100 - accuracy: 0.5855 - val_loss: 2.0080 - val_accuracy: 0.5684 Epoch 172/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8015 - accuracy: 0.5856 - val_loss: 2.0027 - val_accuracy: 0.5674 Epoch 173/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7915 - accuracy: 0.5870 - val_loss: 1.9999 - val_accuracy: 0.5656 Epoch 174/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7909 - accuracy: 0.5892 - val_loss: 1.9903 - val_accuracy: 0.5670 Epoch 175/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7803 - accuracy: 0.5885 - val_loss: 1.9776 - val_accuracy: 0.5713 Epoch 176/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7676 - accuracy: 0.5921 - val_loss: 1.9724 - val_accuracy: 0.5708 Epoch 177/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7620 - accuracy: 0.5954 - val_loss: 1.9674 - val_accuracy: 0.5718 Epoch 178/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7564 - accuracy: 0.5939 - val_loss: 1.9594 - val_accuracy: 0.5732 Epoch 179/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7506 - accuracy: 0.5953 - val_loss: 1.9531 - val_accuracy: 0.5765 Epoch 180/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7382 - accuracy: 0.5980 - val_loss: 1.9489 - val_accuracy: 0.5751 Epoch 181/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7341 - accuracy: 0.5994 - val_loss: 1.9396 - val_accuracy: 0.5793 Epoch 182/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7268 - accuracy: 0.6004 - val_loss: 1.9340 - val_accuracy: 0.5783 Epoch 183/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7191 - accuracy: 0.5999 - val_loss: 1.9284 - val_accuracy: 0.5798 Epoch 184/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7123 - accuracy: 0.6032 - val_loss: 1.9253 - val_accuracy: 0.5784 Epoch 185/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7050 - accuracy: 0.6044 - val_loss: 1.9151 - val_accuracy: 0.5808 Epoch 186/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6980 - accuracy: 0.6071 - val_loss: 1.9110 - val_accuracy: 0.5789 Epoch 187/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6866 - accuracy: 0.6092 - val_loss: 1.9064 - val_accuracy: 0.5859 Epoch 188/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6875 - accuracy: 0.6072 - val_loss: 1.8993 - val_accuracy: 0.5830 Epoch 189/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6836 - accuracy: 0.6088 - val_loss: 1.8892 - val_accuracy: 0.5856 Epoch 190/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6743 - accuracy: 0.6096 - val_loss: 1.8865 - val_accuracy: 0.5865 Epoch 191/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6669 - accuracy: 0.6142 - val_loss: 1.8826 - val_accuracy: 0.5870 Epoch 192/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6604 - accuracy: 0.6132 - val_loss: 1.8734 - val_accuracy: 0.5881 Epoch 193/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6554 - accuracy: 0.6126 - val_loss: 1.8743 - val_accuracy: 0.5882 Epoch 194/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6470 - accuracy: 0.6187 - val_loss: 1.8691 - val_accuracy: 0.5892 Epoch 195/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6508 - accuracy: 0.6134 - val_loss: 1.8550 - val_accuracy: 0.5964 Epoch 196/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6355 - accuracy: 0.6213 - val_loss: 1.8540 - val_accuracy: 0.5943 Epoch 197/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6292 - accuracy: 0.6214 - val_loss: 1.8498 - val_accuracy: 0.5928 Epoch 198/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6241 - accuracy: 0.6221 - val_loss: 1.8454 - val_accuracy: 0.5956 Epoch 199/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6173 - accuracy: 0.6227 - val_loss: 1.8395 - val_accuracy: 0.5983 Epoch 200/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6185 - accuracy: 0.6206 - val_loss: 1.8335 - val_accuracy: 0.5980
Observations
GRU_SGD.summary()
Model: "GRU_SGD"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 256) 205824
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 525,957
Trainable params: 525,957
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_SGD_history.history)
Observations
GRU_SGD.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 2ms/step - loss: 1.8300 - accuracy: 0.6006
[1.830045461654663, 0.600604772567749]
Observations
calc_bleu_and_perplexity("GRU with SGD", GRU_SGD, seed_texts, 0.1)
embrace each day with the world that take life into the earth that BLEU Score: 0.2710 Perplexity: 3.3944 radiate some kindness and let it be the foundation of your soul BLEU Score: 0.6764 Perplexity: 1.8476 believe that this morning brings your way and new beginnings reminding us BLEU Score: 0.0317 Perplexity: 1.9777 life's actual purpose is a step towards excellence and progress and strength to create BLEU Score: 0.4833 Perplexity: 2.2055 dance through each and every take flight of our planet deepest desires a reminder of BLEU Score: 0.0849 Perplexity: 1.9626 let your time and energy will follow up the world of life around you of BLEU Score: 0.0610 Perplexity: 1.8456 every person is a testament to the beauty of our planet and aspirations BLEU Score: 0.6381 Perplexity: 1.7315 our country Singapore is a testament to the beauty and richness that meaningful connections BLEU Score: 0.7607 Perplexity: 1.1745 planet earth is a gift a stroke of the brush a deliberate mark BLEU Score: 0.7144 Perplexity: 2.0510 morning and evening would make it can transform up the world of life we witness the BLEU Score: 0.0649 Perplexity: 2.0196
Observations
tf.keras.backend.clear_session()
# Create the model
GRU_l2 = Sequential(
name='GRU_L2_Regularizer',
layers=[
Embedding(total_words_rolling, 10, input_length = max_sequence_rolling_len),
GRU(256, activation='tanh', kernel_regularizer=l2(0.01)),
Dropout(0.5),
Dense(total_words_rolling, activation='softmax')
]
)
opt = Adam(learning_rate=0.001)
GRU_l2.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
GRU_l2_history = GRU_l2.fit(
X_train_roll, y_train_roll,
epochs=200,
validation_data = (X_val_roll, y_val_roll),
batch_size=32,
verbose=1
)
Epoch 1/200 1148/1148 [==============================] - 7s 6ms/step - loss: 5.2198 - accuracy: 0.1135 - val_loss: 4.7189 - val_accuracy: 0.1478 Epoch 2/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.4693 - accuracy: 0.1727 - val_loss: 4.2076 - val_accuracy: 0.2036 Epoch 3/200 1148/1148 [==============================] - 6s 5ms/step - loss: 4.0053 - accuracy: 0.2234 - val_loss: 3.8017 - val_accuracy: 0.2565 Epoch 4/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.5961 - accuracy: 0.2878 - val_loss: 3.4021 - val_accuracy: 0.3293 Epoch 5/200 1148/1148 [==============================] - 6s 5ms/step - loss: 3.2344 - accuracy: 0.3466 - val_loss: 3.0770 - val_accuracy: 0.3833 Epoch 6/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.9542 - accuracy: 0.3931 - val_loss: 2.8476 - val_accuracy: 0.4269 Epoch 7/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.7423 - accuracy: 0.4256 - val_loss: 2.6940 - val_accuracy: 0.4452 Epoch 8/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.5757 - accuracy: 0.4533 - val_loss: 2.5554 - val_accuracy: 0.4716 Epoch 9/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.4469 - accuracy: 0.4749 - val_loss: 2.4589 - val_accuracy: 0.4952 Epoch 10/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.3526 - accuracy: 0.4931 - val_loss: 2.3656 - val_accuracy: 0.5086 Epoch 11/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.2597 - accuracy: 0.5094 - val_loss: 2.3127 - val_accuracy: 0.5168 Epoch 12/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1797 - accuracy: 0.5239 - val_loss: 2.2556 - val_accuracy: 0.5295 Epoch 13/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.1181 - accuracy: 0.5347 - val_loss: 2.2144 - val_accuracy: 0.5321 Epoch 14/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0603 - accuracy: 0.5468 - val_loss: 2.1720 - val_accuracy: 0.5416 Epoch 15/200 1148/1148 [==============================] - 6s 5ms/step - loss: 2.0106 - accuracy: 0.5554 - val_loss: 2.1386 - val_accuracy: 0.5526 Epoch 16/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.9606 - accuracy: 0.5666 - val_loss: 2.0998 - val_accuracy: 0.5556 Epoch 17/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.9324 - accuracy: 0.5707 - val_loss: 2.0876 - val_accuracy: 0.5555 Epoch 18/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8871 - accuracy: 0.5814 - val_loss: 2.0597 - val_accuracy: 0.5645 Epoch 19/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8457 - accuracy: 0.5889 - val_loss: 2.0439 - val_accuracy: 0.5695 Epoch 20/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.8199 - accuracy: 0.5948 - val_loss: 2.0084 - val_accuracy: 0.5758 Epoch 21/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7850 - accuracy: 0.6005 - val_loss: 1.9935 - val_accuracy: 0.5805 Epoch 22/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.7589 - accuracy: 0.6079 - val_loss: 1.9694 - val_accuracy: 0.5840 Epoch 23/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.7309 - accuracy: 0.6123 - val_loss: 1.9638 - val_accuracy: 0.5880 Epoch 24/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.7091 - accuracy: 0.6173 - val_loss: 1.9418 - val_accuracy: 0.5919 Epoch 25/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.6758 - accuracy: 0.6243 - val_loss: 1.9165 - val_accuracy: 0.5985 Epoch 26/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.6538 - accuracy: 0.6319 - val_loss: 1.9025 - val_accuracy: 0.5990 Epoch 27/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.6359 - accuracy: 0.6312 - val_loss: 1.8825 - val_accuracy: 0.5982 Epoch 28/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.6093 - accuracy: 0.6382 - val_loss: 1.8793 - val_accuracy: 0.6040 Epoch 29/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.5979 - accuracy: 0.6390 - val_loss: 1.8651 - val_accuracy: 0.6046 Epoch 30/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5675 - accuracy: 0.6472 - val_loss: 1.8508 - val_accuracy: 0.6117 Epoch 31/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5586 - accuracy: 0.6467 - val_loss: 1.8428 - val_accuracy: 0.6109 Epoch 32/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.5364 - accuracy: 0.6542 - val_loss: 1.8267 - val_accuracy: 0.6165 Epoch 33/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.5173 - accuracy: 0.6592 - val_loss: 1.8151 - val_accuracy: 0.6162 Epoch 34/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.5096 - accuracy: 0.6568 - val_loss: 1.8084 - val_accuracy: 0.6219 Epoch 35/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.4814 - accuracy: 0.6656 - val_loss: 1.7931 - val_accuracy: 0.6228 Epoch 36/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.4731 - accuracy: 0.6645 - val_loss: 1.7837 - val_accuracy: 0.6312 Epoch 37/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4609 - accuracy: 0.6709 - val_loss: 1.7704 - val_accuracy: 0.6310 Epoch 38/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4482 - accuracy: 0.6729 - val_loss: 1.7708 - val_accuracy: 0.6335 Epoch 39/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4374 - accuracy: 0.6766 - val_loss: 1.7681 - val_accuracy: 0.6302 Epoch 40/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4179 - accuracy: 0.6784 - val_loss: 1.7548 - val_accuracy: 0.6343 Epoch 41/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.4019 - accuracy: 0.6822 - val_loss: 1.7454 - val_accuracy: 0.6371 Epoch 42/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.3944 - accuracy: 0.6836 - val_loss: 1.7420 - val_accuracy: 0.6367 Epoch 43/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3861 - accuracy: 0.6843 - val_loss: 1.7357 - val_accuracy: 0.6402 Epoch 44/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3754 - accuracy: 0.6855 - val_loss: 1.7193 - val_accuracy: 0.6402 Epoch 45/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3608 - accuracy: 0.6913 - val_loss: 1.7190 - val_accuracy: 0.6445 Epoch 46/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3503 - accuracy: 0.6912 - val_loss: 1.7081 - val_accuracy: 0.6469 Epoch 47/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3407 - accuracy: 0.6938 - val_loss: 1.7023 - val_accuracy: 0.6469 Epoch 48/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3380 - accuracy: 0.6951 - val_loss: 1.7099 - val_accuracy: 0.6456 Epoch 49/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3134 - accuracy: 0.6989 - val_loss: 1.6897 - val_accuracy: 0.6483 Epoch 50/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3189 - accuracy: 0.6980 - val_loss: 1.6969 - val_accuracy: 0.6474 Epoch 51/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.3002 - accuracy: 0.7036 - val_loss: 1.6822 - val_accuracy: 0.6522 Epoch 52/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2902 - accuracy: 0.7052 - val_loss: 1.6637 - val_accuracy: 0.6528 Epoch 53/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2869 - accuracy: 0.7058 - val_loss: 1.6733 - val_accuracy: 0.6537 Epoch 54/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2876 - accuracy: 0.7038 - val_loss: 1.6719 - val_accuracy: 0.6536 Epoch 55/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2710 - accuracy: 0.7075 - val_loss: 1.6666 - val_accuracy: 0.6532 Epoch 56/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2667 - accuracy: 0.7061 - val_loss: 1.6654 - val_accuracy: 0.6580 Epoch 57/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2597 - accuracy: 0.7105 - val_loss: 1.6539 - val_accuracy: 0.6621 Epoch 58/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2527 - accuracy: 0.7119 - val_loss: 1.6526 - val_accuracy: 0.6590 Epoch 59/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2425 - accuracy: 0.7148 - val_loss: 1.6456 - val_accuracy: 0.6602 Epoch 60/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2409 - accuracy: 0.7126 - val_loss: 1.6409 - val_accuracy: 0.6606 Epoch 61/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2308 - accuracy: 0.7174 - val_loss: 1.6436 - val_accuracy: 0.6594 Epoch 62/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.2265 - accuracy: 0.7158 - val_loss: 1.6280 - val_accuracy: 0.6621 Epoch 63/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2205 - accuracy: 0.7182 - val_loss: 1.6287 - val_accuracy: 0.6629 Epoch 64/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2175 - accuracy: 0.7192 - val_loss: 1.6338 - val_accuracy: 0.6635 Epoch 65/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.2664 - accuracy: 0.7088 - val_loss: 1.6337 - val_accuracy: 0.6638 Epoch 66/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.1937 - accuracy: 0.7258 - val_loss: 1.6121 - val_accuracy: 0.6668 Epoch 67/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1832 - accuracy: 0.7250 - val_loss: 1.6242 - val_accuracy: 0.6645 Epoch 68/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1853 - accuracy: 0.7256 - val_loss: 1.6084 - val_accuracy: 0.6676 Epoch 69/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.1834 - accuracy: 0.7269 - val_loss: 1.6191 - val_accuracy: 0.6668 Epoch 70/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1848 - accuracy: 0.7266 - val_loss: 1.6073 - val_accuracy: 0.6651 Epoch 71/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1778 - accuracy: 0.7262 - val_loss: 1.6009 - val_accuracy: 0.6686 Epoch 72/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1755 - accuracy: 0.7261 - val_loss: 1.6156 - val_accuracy: 0.6665 Epoch 73/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1733 - accuracy: 0.7286 - val_loss: 1.6015 - val_accuracy: 0.6704 Epoch 74/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1674 - accuracy: 0.7259 - val_loss: 1.5915 - val_accuracy: 0.6697 Epoch 75/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1570 - accuracy: 0.7287 - val_loss: 1.6001 - val_accuracy: 0.6675 Epoch 76/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1570 - accuracy: 0.7303 - val_loss: 1.6045 - val_accuracy: 0.6703 Epoch 77/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1566 - accuracy: 0.7297 - val_loss: 1.5915 - val_accuracy: 0.6738 Epoch 78/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.1580 - accuracy: 0.7291 - val_loss: 1.5793 - val_accuracy: 0.6723 Epoch 79/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1471 - accuracy: 0.7313 - val_loss: 1.5789 - val_accuracy: 0.6738 Epoch 80/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1413 - accuracy: 0.7323 - val_loss: 1.5826 - val_accuracy: 0.6717 Epoch 81/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1478 - accuracy: 0.7336 - val_loss: 1.5723 - val_accuracy: 0.6719 Epoch 82/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1296 - accuracy: 0.7338 - val_loss: 1.5784 - val_accuracy: 0.6753 Epoch 83/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1335 - accuracy: 0.7349 - val_loss: 1.5793 - val_accuracy: 0.6726 Epoch 84/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1267 - accuracy: 0.7366 - val_loss: 1.5626 - val_accuracy: 0.6734 Epoch 85/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1236 - accuracy: 0.7366 - val_loss: 1.5633 - val_accuracy: 0.6754 Epoch 86/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1220 - accuracy: 0.7365 - val_loss: 1.5675 - val_accuracy: 0.6737 Epoch 87/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1216 - accuracy: 0.7354 - val_loss: 1.5566 - val_accuracy: 0.6786 Epoch 88/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1194 - accuracy: 0.7360 - val_loss: 1.5624 - val_accuracy: 0.6770 Epoch 89/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.1100 - accuracy: 0.7400 - val_loss: 1.5588 - val_accuracy: 0.6767 Epoch 90/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1187 - accuracy: 0.7365 - val_loss: 1.5494 - val_accuracy: 0.6756 Epoch 91/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1033 - accuracy: 0.7391 - val_loss: 1.5445 - val_accuracy: 0.6761 Epoch 92/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0999 - accuracy: 0.7408 - val_loss: 1.5524 - val_accuracy: 0.6775 Epoch 93/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1027 - accuracy: 0.7379 - val_loss: 1.5339 - val_accuracy: 0.6794 Epoch 94/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.1004 - accuracy: 0.7377 - val_loss: 1.5436 - val_accuracy: 0.6769 Epoch 95/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0953 - accuracy: 0.7423 - val_loss: 1.5436 - val_accuracy: 0.6796 Epoch 96/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0884 - accuracy: 0.7418 - val_loss: 1.5463 - val_accuracy: 0.6793 Epoch 97/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0888 - accuracy: 0.7428 - val_loss: 1.5248 - val_accuracy: 0.6805 Epoch 98/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0899 - accuracy: 0.7434 - val_loss: 1.5311 - val_accuracy: 0.6811 Epoch 99/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0806 - accuracy: 0.7442 - val_loss: 1.5293 - val_accuracy: 0.6809 Epoch 100/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0816 - accuracy: 0.7453 - val_loss: 1.5351 - val_accuracy: 0.6763 Epoch 101/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0756 - accuracy: 0.7452 - val_loss: 1.5329 - val_accuracy: 0.6782 Epoch 102/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0772 - accuracy: 0.7469 - val_loss: 1.5284 - val_accuracy: 0.6831 Epoch 103/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0715 - accuracy: 0.7455 - val_loss: 1.5277 - val_accuracy: 0.6775 Epoch 104/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0764 - accuracy: 0.7446 - val_loss: 1.5201 - val_accuracy: 0.6846 Epoch 105/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0748 - accuracy: 0.7470 - val_loss: 1.5182 - val_accuracy: 0.6828 Epoch 106/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0629 - accuracy: 0.7471 - val_loss: 1.5175 - val_accuracy: 0.6844 Epoch 107/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0693 - accuracy: 0.7459 - val_loss: 1.5300 - val_accuracy: 0.6807 Epoch 108/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0564 - accuracy: 0.7494 - val_loss: 1.5234 - val_accuracy: 0.6836 Epoch 109/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0559 - accuracy: 0.7478 - val_loss: 1.5231 - val_accuracy: 0.6837 Epoch 110/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0552 - accuracy: 0.7492 - val_loss: 1.5092 - val_accuracy: 0.6850 Epoch 111/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0533 - accuracy: 0.7498 - val_loss: 1.5041 - val_accuracy: 0.6837 Epoch 112/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0509 - accuracy: 0.7496 - val_loss: 1.5133 - val_accuracy: 0.6845 Epoch 113/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0582 - accuracy: 0.7480 - val_loss: 1.5075 - val_accuracy: 0.6866 Epoch 114/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0484 - accuracy: 0.7491 - val_loss: 1.5115 - val_accuracy: 0.6835 Epoch 115/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0482 - accuracy: 0.7491 - val_loss: 1.5036 - val_accuracy: 0.6867 Epoch 116/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0506 - accuracy: 0.7506 - val_loss: 1.5139 - val_accuracy: 0.6851 Epoch 117/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0429 - accuracy: 0.7495 - val_loss: 1.5096 - val_accuracy: 0.6855 Epoch 118/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0444 - accuracy: 0.7477 - val_loss: 1.5048 - val_accuracy: 0.6876 Epoch 119/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0357 - accuracy: 0.7530 - val_loss: 1.4985 - val_accuracy: 0.6854 Epoch 120/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0359 - accuracy: 0.7534 - val_loss: 1.4918 - val_accuracy: 0.6854 Epoch 121/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0307 - accuracy: 0.7502 - val_loss: 1.5023 - val_accuracy: 0.6862 Epoch 122/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0399 - accuracy: 0.7515 - val_loss: 1.4920 - val_accuracy: 0.6858 Epoch 123/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0307 - accuracy: 0.7531 - val_loss: 1.4868 - val_accuracy: 0.6895 Epoch 124/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0253 - accuracy: 0.7555 - val_loss: 1.4972 - val_accuracy: 0.6873 Epoch 125/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0268 - accuracy: 0.7548 - val_loss: 1.4977 - val_accuracy: 0.6874 Epoch 126/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0297 - accuracy: 0.7535 - val_loss: 1.4976 - val_accuracy: 0.6897 Epoch 127/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0282 - accuracy: 0.7519 - val_loss: 1.4846 - val_accuracy: 0.6888 Epoch 128/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0157 - accuracy: 0.7558 - val_loss: 1.4850 - val_accuracy: 0.6894 Epoch 129/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0184 - accuracy: 0.7543 - val_loss: 1.4997 - val_accuracy: 0.6889 Epoch 130/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0231 - accuracy: 0.7529 - val_loss: 1.4841 - val_accuracy: 0.6877 Epoch 131/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0158 - accuracy: 0.7572 - val_loss: 1.4842 - val_accuracy: 0.6882 Epoch 132/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0210 - accuracy: 0.7524 - val_loss: 1.4854 - val_accuracy: 0.6903 Epoch 133/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0179 - accuracy: 0.7549 - val_loss: 1.4833 - val_accuracy: 0.6887 Epoch 134/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0175 - accuracy: 0.7556 - val_loss: 1.4773 - val_accuracy: 0.6890 Epoch 135/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0103 - accuracy: 0.7573 - val_loss: 1.4713 - val_accuracy: 0.6906 Epoch 136/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0214 - accuracy: 0.7542 - val_loss: 1.4826 - val_accuracy: 0.6880 Epoch 137/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0091 - accuracy: 0.7567 - val_loss: 1.4661 - val_accuracy: 0.6881 Epoch 138/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0092 - accuracy: 0.7569 - val_loss: 1.4810 - val_accuracy: 0.6872 Epoch 139/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0100 - accuracy: 0.7569 - val_loss: 1.4729 - val_accuracy: 0.6893 Epoch 140/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0041 - accuracy: 0.7569 - val_loss: 1.4675 - val_accuracy: 0.6926 Epoch 141/200 1148/1148 [==============================] - 7s 6ms/step - loss: 1.0072 - accuracy: 0.7580 - val_loss: 1.4703 - val_accuracy: 0.6924 Epoch 142/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0032 - accuracy: 0.7568 - val_loss: 1.4730 - val_accuracy: 0.6895 Epoch 143/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0022 - accuracy: 0.7584 - val_loss: 1.4672 - val_accuracy: 0.6906 Epoch 144/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0011 - accuracy: 0.7570 - val_loss: 1.4753 - val_accuracy: 0.6899 Epoch 145/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0029 - accuracy: 0.7564 - val_loss: 1.4729 - val_accuracy: 0.6886 Epoch 146/200 1148/1148 [==============================] - 6s 5ms/step - loss: 1.0064 - accuracy: 0.7554 - val_loss: 1.4804 - val_accuracy: 0.6894 Epoch 147/200 1148/1148 [==============================] - 6s 6ms/step - loss: 1.0123 - accuracy: 0.7582 - val_loss: 1.4826 - val_accuracy: 0.6874 Epoch 148/200 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9965 - accuracy: 0.7589 - val_loss: 1.4601 - val_accuracy: 0.6925 Epoch 149/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9987 - accuracy: 0.7569 - val_loss: 1.4669 - val_accuracy: 0.6921 Epoch 150/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9951 - accuracy: 0.7600 - val_loss: 1.4626 - val_accuracy: 0.6922 Epoch 151/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9952 - accuracy: 0.7589 - val_loss: 1.4703 - val_accuracy: 0.6889 Epoch 152/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9953 - accuracy: 0.7588 - val_loss: 1.4715 - val_accuracy: 0.6902 Epoch 153/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9870 - accuracy: 0.7593 - val_loss: 1.4575 - val_accuracy: 0.6918 Epoch 154/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9942 - accuracy: 0.7573 - val_loss: 1.4706 - val_accuracy: 0.6926 Epoch 155/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9900 - accuracy: 0.7605 - val_loss: 1.4567 - val_accuracy: 0.6939 Epoch 156/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9924 - accuracy: 0.7580 - val_loss: 1.4725 - val_accuracy: 0.6893 Epoch 157/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9896 - accuracy: 0.7605 - val_loss: 1.4665 - val_accuracy: 0.6922 Epoch 158/200 1148/1148 [==============================] - 7s 6ms/step - loss: 0.9946 - accuracy: 0.7597 - val_loss: 1.4532 - val_accuracy: 0.6966 Epoch 159/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9903 - accuracy: 0.7593 - val_loss: 1.4568 - val_accuracy: 0.6939 Epoch 160/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9868 - accuracy: 0.7590 - val_loss: 1.4544 - val_accuracy: 0.6950 Epoch 161/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9779 - accuracy: 0.7613 - val_loss: 1.4593 - val_accuracy: 0.6933 Epoch 162/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9815 - accuracy: 0.7610 - val_loss: 1.4488 - val_accuracy: 0.6952 Epoch 163/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9930 - accuracy: 0.7599 - val_loss: 1.4573 - val_accuracy: 0.6951 Epoch 164/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9864 - accuracy: 0.7600 - val_loss: 1.4566 - val_accuracy: 0.6934 Epoch 165/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9877 - accuracy: 0.7601 - val_loss: 1.4500 - val_accuracy: 0.6952 Epoch 166/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9785 - accuracy: 0.7621 - val_loss: 1.4609 - val_accuracy: 0.6969 Epoch 167/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9821 - accuracy: 0.7607 - val_loss: 1.4635 - val_accuracy: 0.6930 Epoch 168/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9829 - accuracy: 0.7589 - val_loss: 1.4481 - val_accuracy: 0.6949 Epoch 169/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9696 - accuracy: 0.7640 - val_loss: 1.4416 - val_accuracy: 0.6966 Epoch 170/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9822 - accuracy: 0.7589 - val_loss: 1.4409 - val_accuracy: 0.6946 Epoch 171/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9829 - accuracy: 0.7615 - val_loss: 1.4482 - val_accuracy: 0.6929 Epoch 172/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9794 - accuracy: 0.7615 - val_loss: 1.4468 - val_accuracy: 0.6955 Epoch 173/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9751 - accuracy: 0.7612 - val_loss: 1.4603 - val_accuracy: 0.6954 Epoch 174/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9919 - accuracy: 0.7584 - val_loss: 1.4590 - val_accuracy: 0.6930 Epoch 175/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9831 - accuracy: 0.7602 - val_loss: 1.4491 - val_accuracy: 0.6958 Epoch 176/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9736 - accuracy: 0.7626 - val_loss: 1.4472 - val_accuracy: 0.6939 Epoch 177/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9795 - accuracy: 0.7619 - val_loss: 1.4552 - val_accuracy: 0.6964 Epoch 178/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9701 - accuracy: 0.7640 - val_loss: 1.4399 - val_accuracy: 0.6977 Epoch 179/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9685 - accuracy: 0.7630 - val_loss: 1.4434 - val_accuracy: 0.6968 Epoch 180/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9691 - accuracy: 0.7632 - val_loss: 1.4239 - val_accuracy: 0.6954 Epoch 181/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9847 - accuracy: 0.7599 - val_loss: 1.4816 - val_accuracy: 0.6894 Epoch 182/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9800 - accuracy: 0.7601 - val_loss: 1.4396 - val_accuracy: 0.6979 Epoch 183/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9684 - accuracy: 0.7629 - val_loss: 1.4434 - val_accuracy: 0.6957 Epoch 184/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9668 - accuracy: 0.7619 - val_loss: 1.4424 - val_accuracy: 0.6961 Epoch 185/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9712 - accuracy: 0.7629 - val_loss: 1.4341 - val_accuracy: 0.6988 Epoch 186/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9683 - accuracy: 0.7630 - val_loss: 1.4442 - val_accuracy: 0.6981 Epoch 187/200 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9730 - accuracy: 0.7635 - val_loss: 1.4473 - val_accuracy: 0.6966 Epoch 188/200 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9670 - accuracy: 0.7650 - val_loss: 1.4449 - val_accuracy: 0.6977 Epoch 189/200 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9706 - accuracy: 0.7625 - val_loss: 1.4463 - val_accuracy: 0.6935 Epoch 190/200 1148/1148 [==============================] - 6s 6ms/step - loss: 0.9661 - accuracy: 0.7626 - val_loss: 1.4485 - val_accuracy: 0.6947 Epoch 191/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9666 - accuracy: 0.7635 - val_loss: 1.4364 - val_accuracy: 0.6985 Epoch 192/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9614 - accuracy: 0.7629 - val_loss: 1.4439 - val_accuracy: 0.6992 Epoch 193/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9726 - accuracy: 0.7630 - val_loss: 1.4373 - val_accuracy: 0.6983 Epoch 194/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9638 - accuracy: 0.7638 - val_loss: 1.4488 - val_accuracy: 0.6963 Epoch 195/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9696 - accuracy: 0.7611 - val_loss: 1.4362 - val_accuracy: 0.7004 Epoch 196/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9630 - accuracy: 0.7639 - val_loss: 1.4321 - val_accuracy: 0.6983 Epoch 197/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9644 - accuracy: 0.7623 - val_loss: 1.4369 - val_accuracy: 0.6987 Epoch 198/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9712 - accuracy: 0.7629 - val_loss: 1.4430 - val_accuracy: 0.6981 Epoch 199/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9727 - accuracy: 0.7608 - val_loss: 1.4354 - val_accuracy: 0.7006 Epoch 200/200 1148/1148 [==============================] - 6s 5ms/step - loss: 0.9587 - accuracy: 0.7652 - val_loss: 1.4266 - val_accuracy: 0.7017
GRU_l2.summary()
Model: "GRU_L2_Regularizer"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 34, 10) 11990
gru (GRU) (None, 256) 205824
dropout (Dropout) (None, 256) 0
dense (Dense) (None, 1199) 308143
=================================================================
Total params: 525,957
Trainable params: 525,957
Non-trainable params: 0
_________________________________________________________________
plot_learning_curve(GRU_l2_history.history)
Observations
GRU_l2.evaluate(X_test_roll, y_test_roll)
383/383 [==============================] - 1s 3ms/step - loss: 1.4379 - accuracy: 0.6980
[1.437853455543518, 0.6980222463607788]
Observations
calc_bleu_and_perplexity("GRU with L2 Regularizer", GRU_l2, seed_texts, 0.1)
embrace each day with a heart full of gratitude for the soul of BLEU Score: 0.7238 Perplexity: 1.3966 radiate some gratitude for it turns even the smallest gifts into treasures BLEU Score: 0.8055 Perplexity: 1.3421 believe that yourself for you are the architect of your destiny each BLEU Score: 0.8125 Perplexity: 1.2718 life's actual purpose is the pursuit of purpose and meaning it southern nature park's BLEU Score: 0.4250 Perplexity: 1.4077 dance through each and every step to a treasure in the treasury of cherished memories BLEU Score: 0.5576 Perplexity: 1.2261 let your time and energy and the opportunities and touch the opportunities that this morning BLEU Score: 0.0552 Perplexity: 1.5238 every person is a step towards excellence and progress and innovation and strength BLEU Score: 0.5679 Perplexity: 1.4008 our country Singapore is created change the love to your growth and compassion your BLEU Score: 0.0635 Perplexity: 2.0135 planet earth is the compass of endless exploration heart they hold the keys BLEU Score: 0.6248 Perplexity: 1.7107 morning and evening would make it can transform lives and inspire change and you will conquer BLEU Score: 0.5043 Perplexity: 1.3311
Observations
#Function to plot a graph to consolidate BLEU scores and perplexity, BERTScore
def calc_scores(model_name ,model, seed_texts, temperature, num_word=10):
bleu_scores = []
perplexity_scores = []
for seed_text in seed_texts:
prediction, loss = predict_next_words(seed_text, model, temperature, num_word)
print(''.join(prediction))
bleu_score = evaluate_bleu_score(prediction)
perplexity = calculate_perplexity(loss, len(prediction.split(' ')))
bert_score = evaluate_bert_score(prediction)
print()
bleu_scores.append(bleu_score)
perplexity_scores.append(perplexity)
# Plot a bar graph to show the BLEU scores
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10,8), sharex=True)
# Plot BLEU scores
ax1.bar(range(1, len(bleu_scores)+1), bleu_scores, color='blue', alpha=0.7)
ax1.set_title(f'BLEU Scores: {model_name}')
ax1.set_ylabel("BLEU Score", color="blue")
ax1.axhline(y=0.5, color='r', linestyle='-')
# Plot perplexity scores
ax2.bar(range(1, len(perplexity_scores)+1), perplexity_scores, color='green', alpha=0.7)
ax2.set_title(f'Perplexity Scores: {model_name}')
ax2.set_ylabel("Perplexity", color="green")
ax2.axhline(y=1.5, color='grey', linestyle='--')
plt.tight_layout()
plt.show()
calc_scores("Final GRU Model", GRU_V2, seed_texts, 0.1)
embrace each day with a heart full of gratitude for it is the BLEU Score: 0.8735 Perplexity: 1.1224
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9212 Recall: 0.9651 F1 Score: 0.9426 radiate some gratitude for it is the heartbeat of a joyful heart BLEU Score: 0.8055 Perplexity: 1.4275
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9435 Recall: 0.9491 F1 Score: 0.9455 believe that yourself and you will be a source of light for BLEU Score: 0.8125 Perplexity: 1.2205
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9359 Recall: 0.9287 F1 Score: 0.9323 life's actual purpose is the pursuit of our passions and dreams and aspirations and BLEU Score: 0.5845 Perplexity: 1.0996
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9127 Recall: 0.9426 F1 Score: 0.9274 dance through each and every inspire is a testament to the beauty of our uniqueness BLEU Score: 0.6125 Perplexity: 1.1297
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9018 Recall: 0.9402 F1 Score: 0.9206 let your time and energy will follow and become a beacon of light in the BLEU Score: 0.5114 Perplexity: 1.3926
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.8814 Recall: 0.9007 F1 Score: 0.8902 every person is the jewels set in the crown of the sea of BLEU Score: 0.7281 Perplexity: 1.1199
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9185 Recall: 0.8905 F1 Score: 0.9042 our country Singapore is a testament to the nation's resilience and unwavering determination and BLEU Score: 0.6961 Perplexity: 1.1576
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9069 Recall: 0.9117 F1 Score: 0.9093 planet earth is a testament to your inner strength and resilience that resonates BLEU Score: 0.5714 Perplexity: 1.3073
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.9130 Recall: 0.9364 F1 Score: 0.9245 morning and evening would make it is the compass of endless discovery soul and inner peace BLEU Score: 0.4312 Perplexity: 1.1988
Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Precision: 0.8675 Recall: 0.8877 F1 Score: 0.8775
Observations
def predict_next_words_top_k(input_text, model, temperature, num_words=1):
log_likelihood = 0.0
for _ in range(num_words):
tokens = tokenizer.texts_to_sequences([input_text])[0]
tokens = pad_sequences([tokens], maxlen=max_sequence_rolling_len, padding='pre')
predicted_prob = model.predict(tokens, verbose=0)[0]
prediction = np.log(predicted_prob) / temperature
exp_preds = np.exp(prediction)
predicted_probs = exp_preds / np.sum(exp_preds)
# Get the top 5 words with the highest probability
top_k_indices = np.argpartition(predicted_probs, -5)[-5:]
chosen_word_index = np.random.choice(top_k_indices, p=predicted_probs[top_k_indices]/np.sum(predicted_probs[top_k_indices]))
predicted_word = tokenizer.index_word[chosen_word_index]
input_text += " " + predicted_word
log_likelihood += np.log2(predicted_prob[chosen_word_index])
return input_text, log_likelihood
#Function to plot a graph to consolidate BLEU scores and perplexity
def calc_bleu_and_perplexity_top_k(model_name ,model, seed_texts, temperature, num_word=10):
bleu_scores = []
perplexity_scores = []
for seed_text in seed_texts:
prediction, loss = predict_next_words(seed_text, model, temperature, num_word)
print(''.join(prediction))
bleu_score = evaluate_bleu_score(prediction)
perplexity = calculate_perplexity(loss, len(prediction.split(' ')))
print()
bleu_scores.append(bleu_score)
perplexity_scores.append(perplexity)
# Plot a bar graph to show the BLEU scores
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10,8), sharex=True)
# Plot BLEU scores
ax1.bar(range(1, len(bleu_scores)+1), bleu_scores, color='blue', alpha=0.7)
ax1.set_title(f'BLEU Scores: {model_name}')
ax1.set_ylabel("BLEU Score", color="blue")
ax1.axhline(y=0.5, color='r', linestyle='-')
# Plot perplexity scores
ax2.bar(range(1, len(perplexity_scores)+1), perplexity_scores, color='green', alpha=0.7)
ax2.set_title(f'Perplexity Scores: {model_name}')
ax2.set_ylabel("Perplexity", color="green")
ax2.axhline(y=1.5, color='grey', linestyle='--')
plt.tight_layout()
plt.show()
calc_bleu_and_perplexity_top_k("Final GRU Attempt 1", GRU_V2, seed_texts, 0.23)
embrace each day with a heart full of gratitude for they are the BLEU Score: 0.7543 Perplexity: 1.1299 radiate some gratitude for it is the heartbeat of a joyful heart BLEU Score: 0.8055 Perplexity: 1.4275 believe that yourself and you will be a source of light for BLEU Score: 0.8125 Perplexity: 1.2205 life's actual purpose is the pursuit of our passions and dreams and aspirations and BLEU Score: 0.5845 Perplexity: 1.0996 dance through each and every inspire is a testament to the beauty of our uniqueness BLEU Score: 0.6125 Perplexity: 1.1297 let your time and energy will follow and become a beacon of light in the BLEU Score: 0.5114 Perplexity: 1.3926 every person is the jewels set in the crown of the sea of BLEU Score: 0.7281 Perplexity: 1.1199 our country Singapore is a testament to the nation's resilience and unwavering determination and BLEU Score: 0.6961 Perplexity: 1.1576 planet earth is a testament to your inner strength and resilience that resonates BLEU Score: 0.5714 Perplexity: 1.3073 morning and evening would make it is the driving force behind your fulfilled life story the BLEU Score: 0.3987 Perplexity: 1.3207
calc_bleu_and_perplexity_top_k("Final GRU Attempt 2", GRU_V2, seed_texts, 0.3)
embrace each day with a heart full of gratitude for they are the BLEU Score: 0.7543 Perplexity: 1.1299 radiate some grace and let it be the signature of your presence BLEU Score: 0.8055 Perplexity: 1.2933 believe that yourself and let your light shine bright brightly park's powerful BLEU Score: 0.5157 Perplexity: 1.5878 life's actual purpose is the pursuit of our passions and dreams and aspirations and BLEU Score: 0.5845 Perplexity: 1.0996 dance through each and every inspire is a testament to the beauty of our uniqueness BLEU Score: 0.6125 Perplexity: 1.1297 let your time and energy will follow the simplest moments into treasures of our planet BLEU Score: 0.3947 Perplexity: 1.4538 every person is the promise of becoming reality and let your heart be BLEU Score: 0.5679 Perplexity: 1.3729 our country Singapore is a testament to the nation's resilience and unwavering determination and BLEU Score: 0.6961 Perplexity: 1.1576 planet earth is a testament to your inner strength and resilience that resonates BLEU Score: 0.5714 Perplexity: 1.3073 morning and evening would make it is the driving force behind your actions be the legacy BLEU Score: 0.6221 Perplexity: 1.3902
Observations
# Save weights of the final model
GRU_V2.save_weights("./RNN_FinalModels/Final_GRU_256.h5")
In summary, for next word prediction, there is insufficient data to make the accuracy as high as models used for other tasks, predictions are also not very coherent because of that. The model I think performed best for this task is the GRU with 256 units and a Dropout of 0.3 based on how coherent the text was, how high the BLEU score was and how low the perplexity values were.